mirror of
https://github.com/StackExchange/dnscontrol.git
synced 2024-09-20 06:46:19 +08:00
ROUTE53: Adopt aws-sdk-go-v2 (#1321)
* Switch to aws-sdk-go-v2 AWS has released v2 of their SDK for Go. See: https://aws.github.io/aws-sdk-go-v2/ One big advantage of this is no longer needing to export the `AWS_SDK_LOAD_CONFIG=1` env var when using named profiles. * Update integration test README * Reenable pager601 and pager1201 integration tests for AWS Route53 * Implement intelligent batching for Route53 record changes The AWS Route53 API for batch record changes limits the request size to the smaller of: - 1000 records. - 32000 characters total for record values. Also UPSERTs count as double (a DELETE and then a CREATE). This commit changes how the record ChangeBatches are created to respect these limits. * Remove old comments Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>
This commit is contained in:
Jaye Doepke
GitHub
parent
899a34ddd4
commit
900d4042e8
|
|
@ -36,10 +36,9 @@ $ export AWS_SESSION_TOKEN=ZZZZZZZZ
|
|||
}
|
||||
{% endhighlight %}
|
||||
|
||||
Alternatively if you want to used [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) you need to export the following variables
|
||||
Alternatively if you want to used [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) you need to export the following variable
|
||||
|
||||
```
|
||||
$ export AWS_SDK_LOAD_CONFIG=1
|
||||
$ export AWS_PROFILE=ZZZZZZZZ
|
||||
```
|
||||
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -11,7 +11,11 @@ require (
|
|||
github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf
|
||||
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
|
||||
github.com/aws/aws-sdk-go v1.42.13
|
||||
github.com/aws/aws-sdk-go-v2 v1.11.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.10.1
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.6.1
|
||||
github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0
|
||||
github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0
|
||||
github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
|
||||
github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e
|
||||
github.com/billputer/go-namecheap v0.0.0-20210108011502-994a912fb7f9
|
||||
|
|
|
|||
29
go.sum
29
go.sum
|
|
@ -99,8 +99,32 @@ github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4
|
|||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
|
||||
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aws/aws-sdk-go v1.42.13 h1:+Nx87T+Bjiq2XybxK6vI98cTEBPLE/hILuZyEenlyEg=
|
||||
github.com/aws/aws-sdk-go v1.42.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||
github.com/aws/aws-sdk-go-v2 v1.11.0 h1:HxyD62DyNhCfiFGUHqJ/xITD6rAjJ7Dm/2nLxLmO4Ag=
|
||||
github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.10.1 h1:z/ViqIjW6ZeuLWgTWMTSyZzaVWo/1cWeVf1Uu+RF01E=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.10.1/go.mod h1:auIv5pIIn3jIBHNRcVQcsczn6Pfa6Dyv80Fai0ueoJU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.6.1 h1:A39JYth2fFCx+omN/gib/jIppx3rRnt2r7UKPq7Mh5Y=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.6.1/go.mod h1:QyvQk1IYTqBWSi1T6UgT/W8DMxBVa5pVuLFSRLLhGf8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0 h1:OpZjuUy8Jt3CA1WgJgBC5Bz+uOjE5Ppx4NFTRaooUuA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0/go.mod h1:5E1J3/TTYy6z909QNR0QnXGBpfESYGDqd3O0zqONghU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0 h1:zY8cNmbBXt3pzjgWgdIbzpQ6qxoCwt+Nx9JbrAf2mbY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0/go.mod h1:NO3Q5ZTTQtO2xIg2+xTXYDiT7knSejfeDm7WGDaOo0U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0 h1:Z3aR/OXBnkYK9zXkNkfitHX6SmUBzSsx8VMHbH4Lvhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0/go.mod h1:anlUzBoEWglcUxUQwZA7HQOEVEnQALVZsizAapB2hq8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0 h1:c10Z7fWxtJCoyc8rv06jdh9xrKnu7bAJiRaKWvTb2mU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0/go.mod h1:6oXGy4GLpypD3uCh8wcqztigGgmhLToMfjavgh+VySg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0 h1:qGZWS/WgiFY+Zgad2u0gwBHpJxz6Ne401JE7iQI1nKs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0/go.mod h1:Mq6AEc+oEjCUlBuLiK5YwW4shSOAKCQ3tXN0sQeYoBA=
|
||||
github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0 h1:0SJgP/L7413/m8itu30tEcsEfup9Ky5TOyhqGaefZ4c=
|
||||
github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0/go.mod h1:s0AHQXKd6Jo4hsu2N9R1kxJuKLsEY8pIp3GUegGMrqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0 h1:wXztJWR5n6LIep/rWo5HlMPjyyUP67xaGDWaFjCcbTU=
|
||||
github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0/go.mod h1:qPnejxOymP2/tcqFuYAWJyaeCgSuEjahjXT5s/2bteI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.6.0 h1:JDgKIUZOmLFu/Rv6zXLrVTWCmzA0jcTdvsT8iFIKrAI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.6.0/go.mod h1:Q/l0ON1annSU+mc0JybDy1Gy6dnJxIcWjphO6qJPzvM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.10.0 h1:1jh8J+JjYRp+QWKOsaZt7rGUgoyrqiiVwIm+w0ymeUw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.10.0/go.mod h1:jLKCFqS+1T4i7HDqCP9GM4Uk75YW1cS0o82LdxpMyOE=
|
||||
github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58=
|
||||
github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
|
||||
github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0=
|
||||
github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
|
|
@ -691,7 +715,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
|
|||
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI=
|
||||
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
|
|
|
|||
|
|
@ -1041,7 +1041,7 @@ func makeTests(t *testing.T) []*TestGroup {
|
|||
//"AZURE_DNS", // Currently failing.
|
||||
"HEXONET",
|
||||
"GCLOUD",
|
||||
//"ROUTE53", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/908
|
||||
"ROUTE53",
|
||||
),
|
||||
tc("601 records", manyA("rec%04d", "1.2.3.4", 600)...),
|
||||
tc("Update 601 records", manyA("rec%04d", "1.2.3.5", 600)...),
|
||||
|
|
@ -1054,7 +1054,7 @@ func makeTests(t *testing.T) []*TestGroup {
|
|||
//"AZURE_DNS", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/770
|
||||
"HEXONET",
|
||||
"HOSTINGDE",
|
||||
//"ROUTE53", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/908
|
||||
"ROUTE53",
|
||||
),
|
||||
tc("1200 records", manyA("rec%04d", "1.2.3.4", 1200)...),
|
||||
tc("Update 1200 records", manyA("rec%04d", "1.2.3.5", 1200)...),
|
||||
|
|
|
|||
|
|
@ -12,19 +12,19 @@ For each step, it will run the config once and expect changes. It will run it ag
|
|||
|
||||
## Running a test
|
||||
|
||||
1. Define all environment variables expected for the provider you wish to run. I setup a local `.env` file with the appropriate values and use [zoo](https://github.com/jsonmaur/zoo) to run my commands.
|
||||
2. run `go test -v -provider $NAME` where $NAME is the name of the provider you wish to run.
|
||||
1. Define all environment variables expected for the provider you wish to run. I setup a local `.env` file with the appropriate values and use [zoo](https://github.com/jsonmaur/zoo) to run my commands.
|
||||
2. run `go test -v -provider $NAME` where $NAME is the name of the provider you wish to run.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
$ egrep R53 providers.json
|
||||
"KeyId": "$R53_KEY_ID",
|
||||
"SecretKey": "$R53_KEY",
|
||||
"domain": "$R53_DOMAIN"
|
||||
$ export R53_KEY_ID="redacted"
|
||||
$ export R53_KEY="also redacted"
|
||||
$ export R53_DOMAIN="testdomain.tld"
|
||||
$ egrep ROUTE53 providers.json
|
||||
"KeyId": "$ROUTE53_KEY_ID",
|
||||
"SecretKey": "$ROUTE53_KEY",
|
||||
"domain": "$ROUTE53_DOMAIN"
|
||||
$ export ROUTE53_KEY_ID="redacted"
|
||||
$ export ROUTE53_KEY="also redacted"
|
||||
$ export ROUTE53_DOMAIN="testdomain.tld"
|
||||
$ go test -v -verbose -provider ROUTE53
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package route53
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
|
@ -8,12 +9,15 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
r53 "github.com/aws/aws-sdk-go/service/route53"
|
||||
r53d "github.com/aws/aws-sdk-go/service/route53domains"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
r53 "github.com/aws/aws-sdk-go-v2/service/route53"
|
||||
r53Types "github.com/aws/aws-sdk-go-v2/service/route53/types"
|
||||
r53d "github.com/aws/aws-sdk-go-v2/service/route53domains"
|
||||
r53dTypes "github.com/aws/aws-sdk-go-v2/service/route53domains/types"
|
||||
|
||||
"github.com/StackExchange/dnscontrol/v3/models"
|
||||
"github.com/StackExchange/dnscontrol/v3/pkg/diff"
|
||||
|
|
@ -22,12 +26,12 @@ import (
|
|||
)
|
||||
|
||||
type route53Provider struct {
|
||||
client *r53.Route53
|
||||
registrar *r53d.Route53Domains
|
||||
client *r53.Client
|
||||
registrar *r53d.Client
|
||||
delegationSet *string
|
||||
zonesById map[string]*r53.HostedZone
|
||||
zonesByDomain map[string]*r53.HostedZone
|
||||
originalRecords []*r53.ResourceRecordSet
|
||||
zonesById map[string]r53Types.HostedZone
|
||||
zonesByDomain map[string]r53Types.HostedZone
|
||||
originalRecords []r53Types.ResourceRecordSet
|
||||
}
|
||||
|
||||
func newRoute53Reg(conf map[string]string) (providers.Registrar, error) {
|
||||
|
|
@ -39,28 +43,31 @@ func newRoute53Dsp(conf map[string]string, metadata json.RawMessage) (providers.
|
|||
}
|
||||
|
||||
func newRoute53(m map[string]string, metadata json.RawMessage) (*route53Provider, error) {
|
||||
keyID, secretKey, tokenID := m["KeyId"], m["SecretKey"], m["Token"]
|
||||
|
||||
// Route53 uses a global endpoint and route53domains
|
||||
// currently only has a single regional endpoint in us-east-1
|
||||
// http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region
|
||||
config := &aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
optFns := []func(*config.LoadOptions) error{
|
||||
// Route53 uses a global endpoint and route53domains
|
||||
// currently only has a single regional endpoint in us-east-1
|
||||
// http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region
|
||||
config.WithRegion("us-east-1"),
|
||||
}
|
||||
|
||||
keyID, secretKey, tokenID := m["KeyId"], m["SecretKey"], m["Token"]
|
||||
// Token is optional and left empty unless required
|
||||
if keyID != "" || secretKey != "" {
|
||||
config.Credentials = credentials.NewStaticCredentials(keyID, secretKey, tokenID)
|
||||
optFns = append(optFns, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(keyID, secretKey, tokenID)))
|
||||
}
|
||||
|
||||
config, err := config.LoadDefaultConfig(context.Background(), optFns...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sess := session.Must(session.NewSession(config))
|
||||
|
||||
var dls *string
|
||||
if val, ok := m["DelegationSet"]; ok {
|
||||
fmt.Printf("ROUTE53 DelegationSet %s configured\n", val)
|
||||
dls = sPtr(val)
|
||||
dls = aws.String(val)
|
||||
}
|
||||
api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess), delegationSet: dls}
|
||||
err := api.getZones()
|
||||
api := &route53Provider{client: r53.NewFromConfig(config), registrar: r53d.NewFromConfig(config), delegationSet: dls}
|
||||
err = api.getZones()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -89,10 +96,6 @@ func init() {
|
|||
providers.RegisterCustomRecordType("R53_ALIAS", "ROUTE53", "")
|
||||
}
|
||||
|
||||
func sPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
func withRetry(f func() error) {
|
||||
const maxRetries = 23
|
||||
// TODO: exponential backoff
|
||||
|
|
@ -128,14 +131,14 @@ func (r *route53Provider) ListZones() ([]string, error) {
|
|||
|
||||
func (r *route53Provider) getZones() error {
|
||||
var nextMarker *string
|
||||
r.zonesByDomain = make(map[string]*r53.HostedZone)
|
||||
r.zonesById = make(map[string]*r53.HostedZone)
|
||||
r.zonesByDomain = make(map[string]r53Types.HostedZone)
|
||||
r.zonesById = make(map[string]r53Types.HostedZone)
|
||||
for {
|
||||
var out *r53.ListHostedZonesOutput
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
inp := &r53.ListHostedZonesInput{Marker: nextMarker}
|
||||
out, err = r.client.ListHostedZones(inp)
|
||||
out, err = r.client.ListHostedZones(context.Background(), inp)
|
||||
return err
|
||||
})
|
||||
if err != nil && strings.Contains(err.Error(), "is not authorized") {
|
||||
|
|
@ -144,9 +147,9 @@ func (r *route53Provider) getZones() error {
|
|||
return err
|
||||
}
|
||||
for _, z := range out.HostedZones {
|
||||
domain := strings.TrimSuffix(*z.Name, ".")
|
||||
domain := strings.TrimSuffix(aws.ToString(z.Name), ".")
|
||||
r.zonesByDomain[domain] = z
|
||||
r.zonesById[parseZoneId(*z.Id)] = z
|
||||
r.zonesById[parseZoneId(aws.ToString(z.Id))] = z
|
||||
}
|
||||
if out.NextMarker != nil {
|
||||
nextMarker = out.NextMarker
|
||||
|
|
@ -182,7 +185,7 @@ func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, e
|
|||
var z *r53.GetHostedZoneOutput
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
z, err = r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id})
|
||||
z, err = r.client.GetHostedZone(context.Background(), &r53.GetHostedZoneInput{Id: zone.Id})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -191,9 +194,7 @@ func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, e
|
|||
|
||||
var nss []string
|
||||
if z.DelegationSet != nil {
|
||||
for _, nsPtr := range z.DelegationSet.NameServers {
|
||||
nss = append(nss, *nsPtr)
|
||||
}
|
||||
nss = z.DelegationSet.NameServers
|
||||
}
|
||||
return models.ToNameservers(nss)
|
||||
}
|
||||
|
|
@ -206,11 +207,11 @@ func (r *route53Provider) GetZoneRecords(domain string) (models.Records, error)
|
|||
return nil, errDomainNoExist{domain}
|
||||
}
|
||||
|
||||
func (r *route53Provider) getZone(dc *models.DomainConfig) (*r53.HostedZone, error) {
|
||||
func (r *route53Provider) getZone(dc *models.DomainConfig) (r53Types.HostedZone, error) {
|
||||
if zoneId, ok := dc.Metadata["zone_id"]; ok {
|
||||
zone, ok := r.zonesById[zoneId]
|
||||
if !ok {
|
||||
return nil, errZoneNoExist{zoneId}
|
||||
return r53Types.HostedZone{}, errZoneNoExist{zoneId}
|
||||
}
|
||||
return zone, nil
|
||||
}
|
||||
|
|
@ -219,10 +220,10 @@ func (r *route53Provider) getZone(dc *models.DomainConfig) (*r53.HostedZone, err
|
|||
return zone, nil
|
||||
}
|
||||
|
||||
return nil, errDomainNoExist{dc.Name}
|
||||
return r53Types.HostedZone{}, errDomainNoExist{dc.Name}
|
||||
}
|
||||
|
||||
func (r *route53Provider) getZoneRecords(zone *r53.HostedZone) (models.Records, error) {
|
||||
func (r *route53Provider) getZoneRecords(zone r53Types.HostedZone) (models.Records, error) {
|
||||
records, err := r.fetchRecordSets(zone.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -321,9 +322,9 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
// we collect all changes into one of two categories now:
|
||||
// pure deletions where we delete an entire record set,
|
||||
// or changes where we upsert an entire record set.
|
||||
dels := []*r53.Change{}
|
||||
dels := []r53Types.Change{}
|
||||
delDesc := []string{}
|
||||
changes := []*r53.Change{}
|
||||
changes := []r53Types.Change{}
|
||||
changeDesc := []string{}
|
||||
|
||||
for _, k := range updateOrder {
|
||||
|
|
@ -332,22 +333,26 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
// indicates we should delete all records at that key.
|
||||
if len(recs) == 0 {
|
||||
// To delete, we submit the original resource set we got from r53.
|
||||
var rrset *r53.ResourceRecordSet
|
||||
var (
|
||||
rrset r53Types.ResourceRecordSet
|
||||
found bool
|
||||
)
|
||||
// Find the original resource set:
|
||||
for _, r := range r.originalRecords {
|
||||
if unescape(r.Name) == k.NameFQDN && (*r.Type == k.Type || k.Type == "R53_ALIAS_"+*r.Type) {
|
||||
if unescape(r.Name) == k.NameFQDN && (string(r.Type) == k.Type || k.Type == "R53_ALIAS_"+string(r.Type)) {
|
||||
rrset = r
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if rrset == nil {
|
||||
if !found {
|
||||
// This should not happen.
|
||||
return nil, fmt.Errorf("no record set found to delete. Name: '%s'. Type: '%s'", k.NameFQDN, k.Type)
|
||||
}
|
||||
// Assemble the change and add it to the list:
|
||||
chg := &r53.Change{
|
||||
Action: sPtr("DELETE"),
|
||||
ResourceRecordSet: rrset,
|
||||
chg := r53Types.Change{
|
||||
Action: r53Types.ChangeActionDelete,
|
||||
ResourceRecordSet: &rrset,
|
||||
}
|
||||
dels = append(dels, chg)
|
||||
delDesc = append(delDesc, strings.Join(namesToUpdate[k], "\n"))
|
||||
|
|
@ -363,10 +368,10 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
}
|
||||
for _, r := range recs {
|
||||
rrset := aliasToRRSet(zone, r)
|
||||
rrset.Name = sPtr(k.NameFQDN)
|
||||
rrset.Name = aws.String(k.NameFQDN)
|
||||
// Assemble the change and add it to the list:
|
||||
chg := &r53.Change{
|
||||
Action: sPtr("UPSERT"),
|
||||
chg := r53Types.Change{
|
||||
Action: r53Types.ChangeActionUpsert,
|
||||
ResourceRecordSet: rrset,
|
||||
}
|
||||
changes = append(changes, chg)
|
||||
|
|
@ -374,22 +379,22 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
}
|
||||
} else {
|
||||
// All other keys combine their updates into one rrset:
|
||||
rrset := &r53.ResourceRecordSet{
|
||||
Name: sPtr(k.NameFQDN),
|
||||
Type: sPtr(k.Type),
|
||||
rrset := &r53Types.ResourceRecordSet{
|
||||
Name: aws.String(k.NameFQDN),
|
||||
Type: r53Types.RRType(k.Type),
|
||||
}
|
||||
for _, r := range recs {
|
||||
val := r.GetTargetCombined()
|
||||
rr := &r53.ResourceRecord{
|
||||
Value: &val,
|
||||
rr := r53Types.ResourceRecord{
|
||||
Value: aws.String(val),
|
||||
}
|
||||
rrset.ResourceRecords = append(rrset.ResourceRecords, rr)
|
||||
i := int64(r.TTL)
|
||||
rrset.TTL = &i // TODO: make sure that ttls are consistent within a set
|
||||
}
|
||||
// Assemble the change and add it to the list:
|
||||
chg := &r53.Change{
|
||||
Action: sPtr("UPSERT"),
|
||||
chg := r53Types.Change{
|
||||
Action: r53Types.ChangeActionUpsert,
|
||||
ResourceRecordSet: rrset,
|
||||
}
|
||||
changes = append(changes, chg)
|
||||
|
|
@ -407,7 +412,7 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
var err error
|
||||
req.HostedZoneId = zone.Id
|
||||
withRetry(func() error {
|
||||
_, err = r.client.ChangeResourceRecordSets(req)
|
||||
_, err = r.client.ChangeResourceRecordSets(context.Background(), req)
|
||||
return err
|
||||
})
|
||||
return err
|
||||
|
|
@ -415,75 +420,67 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
|
|||
})
|
||||
}
|
||||
|
||||
getBatchSize := func(size, max int) int {
|
||||
if size > max {
|
||||
return max
|
||||
batcher := newChangeBatcher(dels)
|
||||
for batcher.Next() {
|
||||
start, end := batcher.Batch()
|
||||
batch := dels[start:end]
|
||||
descBatchStr := "\n" + strings.Join(delDesc[start:end], "\n") + "\n"
|
||||
req := &r53.ChangeResourceRecordSetsInput{
|
||||
ChangeBatch: &r53Types.ChangeBatch{Changes: batch},
|
||||
}
|
||||
return size
|
||||
addCorrection(descBatchStr, req)
|
||||
}
|
||||
if err := batcher.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for len(dels) > 0 {
|
||||
batchSize := getBatchSize(len(dels), 1000)
|
||||
batch := dels[:batchSize]
|
||||
dels = dels[batchSize:]
|
||||
delDescBatch := delDesc[:batchSize]
|
||||
delDesc = delDesc[batchSize:]
|
||||
|
||||
delDescBatchStr := "\n" + strings.Join(delDescBatch, "\n") + "\n"
|
||||
|
||||
delReq := &r53.ChangeResourceRecordSetsInput{
|
||||
ChangeBatch: &r53.ChangeBatch{Changes: batch},
|
||||
batcher = newChangeBatcher(changes)
|
||||
for batcher.Next() {
|
||||
start, end := batcher.Batch()
|
||||
batch := changes[start:end]
|
||||
descBatchStr := "\n" + strings.Join(changeDesc[start:end], "\n") + "\n"
|
||||
req := &r53.ChangeResourceRecordSetsInput{
|
||||
ChangeBatch: &r53Types.ChangeBatch{Changes: batch},
|
||||
}
|
||||
addCorrection(delDescBatchStr, delReq)
|
||||
addCorrection(descBatchStr, req)
|
||||
}
|
||||
|
||||
for len(changes) > 0 {
|
||||
batchSize := getBatchSize(len(changes), 500)
|
||||
batch := changes[:batchSize]
|
||||
changes = changes[batchSize:]
|
||||
changeDescBatch := changeDesc[:batchSize]
|
||||
changeDesc = changeDesc[batchSize:]
|
||||
changeDescBatchStr := "\n" + strings.Join(changeDescBatch, "\n") + "\n"
|
||||
|
||||
changeReq := &r53.ChangeResourceRecordSetsInput{
|
||||
ChangeBatch: &r53.ChangeBatch{Changes: batch},
|
||||
}
|
||||
addCorrection(changeDescBatchStr, changeReq)
|
||||
if err := batcher.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return corrections, nil
|
||||
|
||||
}
|
||||
|
||||
func nativeToRecords(set *r53.ResourceRecordSet, origin string) ([]*models.RecordConfig, error) {
|
||||
func nativeToRecords(set r53Types.ResourceRecordSet, origin string) ([]*models.RecordConfig, error) {
|
||||
results := []*models.RecordConfig{}
|
||||
if set.AliasTarget != nil {
|
||||
rc := &models.RecordConfig{
|
||||
Type: "R53_ALIAS",
|
||||
TTL: 300,
|
||||
R53Alias: map[string]string{
|
||||
"type": *set.Type,
|
||||
"zone_id": *set.AliasTarget.HostedZoneId,
|
||||
"type": string(set.Type),
|
||||
"zone_id": aws.ToString(set.AliasTarget.HostedZoneId),
|
||||
},
|
||||
}
|
||||
rc.SetLabelFromFQDN(unescape(set.Name), origin)
|
||||
rc.SetTarget(aws.StringValue(set.AliasTarget.DNSName))
|
||||
rc.SetTarget(aws.ToString(set.AliasTarget.DNSName))
|
||||
results = append(results, rc)
|
||||
} else if set.TrafficPolicyInstanceId != nil {
|
||||
// skip traffic policy records
|
||||
} else {
|
||||
for _, rec := range set.ResourceRecords {
|
||||
switch rtype := *set.Type; rtype {
|
||||
case "SOA":
|
||||
switch rtype := set.Type; rtype {
|
||||
case r53Types.RRTypeSoa:
|
||||
continue
|
||||
case "SPF":
|
||||
case r53Types.RRTypeSpf:
|
||||
// route53 uses a custom record type for SPF
|
||||
rtype = "TXT"
|
||||
fallthrough
|
||||
default:
|
||||
rc := &models.RecordConfig{TTL: uint32(*set.TTL)}
|
||||
rc := &models.RecordConfig{TTL: uint32(aws.ToInt64(set.TTL))}
|
||||
rc.SetLabelFromFQDN(unescape(set.Name), origin)
|
||||
if err := rc.PopulateFromString(rtype, *rec.Value, origin); err != nil {
|
||||
if err := rc.PopulateFromString(string(rtype), *rec.Value, origin); err != nil {
|
||||
return nil, fmt.Errorf("unparsable record received from R53: %w", err)
|
||||
}
|
||||
results = append(results, rc)
|
||||
|
|
@ -500,25 +497,24 @@ func getAliasMap(r *models.RecordConfig) map[string]string {
|
|||
return r.R53Alias
|
||||
}
|
||||
|
||||
func aliasToRRSet(zone *r53.HostedZone, r *models.RecordConfig) *r53.ResourceRecordSet {
|
||||
func aliasToRRSet(zone r53Types.HostedZone, r *models.RecordConfig) *r53Types.ResourceRecordSet {
|
||||
target := r.GetTargetField()
|
||||
zoneID := getZoneID(zone, r)
|
||||
targetHealth := false
|
||||
rrset := &r53.ResourceRecordSet{
|
||||
Type: sPtr(r.R53Alias["type"]),
|
||||
AliasTarget: &r53.AliasTarget{
|
||||
rrset := &r53Types.ResourceRecordSet{
|
||||
Type: r53Types.RRType(r.R53Alias["type"]),
|
||||
AliasTarget: &r53Types.AliasTarget{
|
||||
DNSName: &target,
|
||||
HostedZoneId: aws.String(zoneID),
|
||||
EvaluateTargetHealth: &targetHealth,
|
||||
EvaluateTargetHealth: false,
|
||||
},
|
||||
}
|
||||
return rrset
|
||||
}
|
||||
|
||||
func getZoneID(zone *r53.HostedZone, r *models.RecordConfig) string {
|
||||
func getZoneID(zone r53Types.HostedZone, r *models.RecordConfig) string {
|
||||
zoneID := r.R53Alias["zone_id"]
|
||||
if zoneID == "" {
|
||||
zoneID = aws.StringValue(zone.Id)
|
||||
zoneID = aws.ToString(zone.Id)
|
||||
}
|
||||
return parseZoneId(zoneID)
|
||||
}
|
||||
|
|
@ -566,7 +562,7 @@ func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string,
|
|||
var domainDetail *r53d.GetDomainDetailOutput
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
domainDetail, err = r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName})
|
||||
domainDetail, err = r.registrar.GetDomainDetail(context.Background(), &r53d.GetDomainDetailInput{DomainName: domainName})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -575,22 +571,24 @@ func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string,
|
|||
|
||||
nameservers := []string{}
|
||||
for _, ns := range domainDetail.Nameservers {
|
||||
nameservers = append(nameservers, *ns.Name)
|
||||
nameservers = append(nameservers, aws.ToString(ns.Name))
|
||||
}
|
||||
|
||||
return nameservers, nil
|
||||
}
|
||||
|
||||
func (r *route53Provider) updateRegistrarNameservers(domainName string, nameservers []string) (*string, error) {
|
||||
servers := []*r53d.Nameserver{}
|
||||
servers := make([]r53dTypes.Nameserver, len(nameservers))
|
||||
for i := range nameservers {
|
||||
servers = append(servers, &r53d.Nameserver{Name: &nameservers[i]})
|
||||
servers[i] = r53dTypes.Nameserver{Name: aws.String(nameservers[i])}
|
||||
}
|
||||
var domainUpdate *r53d.UpdateDomainNameserversOutput
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
domainUpdate, err = r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{
|
||||
DomainName: &domainName, Nameservers: servers})
|
||||
domainUpdate, err = r.registrar.UpdateDomainNameservers(context.Background(), &r53d.UpdateDomainNameserversInput{
|
||||
DomainName: aws.String(domainName),
|
||||
Nameservers: servers,
|
||||
})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -600,24 +598,24 @@ func (r *route53Provider) updateRegistrarNameservers(domainName string, nameserv
|
|||
return domainUpdate.OperationId, nil
|
||||
}
|
||||
|
||||
func (r *route53Provider) fetchRecordSets(zoneID *string) ([]*r53.ResourceRecordSet, error) {
|
||||
func (r *route53Provider) fetchRecordSets(zoneID *string) ([]r53Types.ResourceRecordSet, error) {
|
||||
if zoneID == nil || *zoneID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var next *string
|
||||
var nextType *string
|
||||
var records []*r53.ResourceRecordSet
|
||||
var nextType r53Types.RRType
|
||||
var records []r53Types.ResourceRecordSet
|
||||
for {
|
||||
listInput := &r53.ListResourceRecordSetsInput{
|
||||
HostedZoneId: zoneID,
|
||||
StartRecordName: next,
|
||||
StartRecordType: nextType,
|
||||
MaxItems: sPtr("100"),
|
||||
MaxItems: aws.Int32(100),
|
||||
}
|
||||
var list *r53.ListResourceRecordSetsOutput
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
list, err = r.client.ListResourceRecordSets(listInput)
|
||||
list, err = r.client.ListResourceRecordSets(context.Background(), listInput)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -657,12 +655,102 @@ func (r *route53Provider) EnsureDomainExists(domain string) error {
|
|||
in := &r53.CreateHostedZoneInput{
|
||||
Name: &domain,
|
||||
DelegationSetId: r.delegationSet,
|
||||
CallerReference: sPtr(fmt.Sprint(time.Now().UnixNano())),
|
||||
CallerReference: aws.String(fmt.Sprint(time.Now().UnixNano())),
|
||||
}
|
||||
var err error
|
||||
withRetry(func() error {
|
||||
_, err := r.client.CreateHostedZone(in)
|
||||
_, err := r.client.CreateHostedZone(context.Background(), in)
|
||||
return err
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// changeBatcher takes a set of r53Types.Changes and turns them into a series of
|
||||
// batches that meet the limits of the ChangeResourceRecordSets API.
|
||||
//
|
||||
// See also: https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets
|
||||
type changeBatcher struct {
|
||||
changes []r53Types.Change
|
||||
|
||||
maxSize int // Max records per request.
|
||||
maxChars int // Max record value characters per request.
|
||||
|
||||
start, end int // Cursors into changes.
|
||||
err error // Populated by Next.
|
||||
}
|
||||
|
||||
// newChangeBatcher returns a new changeBatcher.
|
||||
func newChangeBatcher(changes []r53Types.Change) *changeBatcher {
|
||||
return &changeBatcher{
|
||||
changes: changes,
|
||||
maxSize: 1000, // "A request cannot contain more than 1,000 ResourceRecord elements."
|
||||
maxChars: 32000, // "The sum of the number of characters (including spaces) in all Value elements in a request cannot exceed 32,000 characters."
|
||||
}
|
||||
}
|
||||
|
||||
// Next returns true if there is another batch of Changes.
|
||||
// It returns false if there are no more batches or an error occurred.
|
||||
func (b *changeBatcher) Next() bool {
|
||||
if b.end >= len(b.changes) || b.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
start, end := b.end, b.end
|
||||
var (
|
||||
reqSize int
|
||||
reqChars int
|
||||
)
|
||||
for end < len(b.changes) {
|
||||
c := &b.changes[end]
|
||||
|
||||
// Check that we won't exceed 1000 ResourceRecords in the request.
|
||||
rrsetSize := len(c.ResourceRecordSet.ResourceRecords)
|
||||
if c.Action == r53Types.ChangeActionUpsert {
|
||||
// "When the value of the Action element is UPSERT, each ResourceRecord element is counted twice."
|
||||
rrsetSize *= 2
|
||||
}
|
||||
if newReqSize := reqSize + rrsetSize; newReqSize > b.maxSize {
|
||||
break
|
||||
} else {
|
||||
reqSize = newReqSize
|
||||
}
|
||||
|
||||
// Check that we won't exceed 32000 Value characters in the request.
|
||||
var rrsetChars int
|
||||
for _, rr := range c.ResourceRecordSet.ResourceRecords {
|
||||
rrsetChars += utf8.RuneCountInString(aws.ToString(rr.Value))
|
||||
}
|
||||
if c.Action == r53Types.ChangeActionUpsert {
|
||||
// "When the value of the Action element is UPSERT, each character in a Value element is counted twice."
|
||||
rrsetChars *= 2
|
||||
}
|
||||
if newReqChars := reqChars + rrsetChars; newReqChars > b.maxChars {
|
||||
break
|
||||
} else {
|
||||
reqChars = newReqChars
|
||||
}
|
||||
|
||||
end++
|
||||
}
|
||||
|
||||
if start == end {
|
||||
b.err = errors.New("could not create ChangeResourceRecordSets request within AWS API limits")
|
||||
return false
|
||||
}
|
||||
|
||||
b.start = start
|
||||
b.end = end
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Batch returns the current batch. It should only be called
|
||||
// after Next returns true.
|
||||
func (b *changeBatcher) Batch() (start, end int) {
|
||||
return b.start, b.end
|
||||
}
|
||||
|
||||
// Err returns the error encountered during the previous call to Next.
|
||||
func (b *changeBatcher) Err() error {
|
||||
return b.err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
package route53
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
r53Types "github.com/aws/aws-sdk-go-v2/service/route53/types"
|
||||
)
|
||||
|
||||
func TestUnescape(t *testing.T) {
|
||||
var tests = []struct {
|
||||
|
|
@ -22,3 +29,115 @@ func TestUnescape(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
type batch struct {
|
||||
start int
|
||||
end int
|
||||
}
|
||||
|
||||
func (b batch) String() string {
|
||||
return fmt.Sprintf("%d:%d", b.start, b.end)
|
||||
}
|
||||
|
||||
func Test_changeBatcher(t *testing.T) {
|
||||
genChanges := func(action r53Types.ChangeAction, typ r53Types.RRType, namePattern string, n int, targets ...string) []r53Types.Change {
|
||||
changes := make([]r53Types.Change, n)
|
||||
for i := 0; i < n; i++ {
|
||||
changes[i].Action = action
|
||||
changes[i].ResourceRecordSet = &r53Types.ResourceRecordSet{
|
||||
Name: aws.String(fmt.Sprintf(namePattern, i)),
|
||||
Type: typ,
|
||||
}
|
||||
for j := 0; j < len(targets); j++ {
|
||||
changes[i].ResourceRecordSet.ResourceRecords = append(changes[i].ResourceRecordSet.ResourceRecords, r53Types.ResourceRecord{
|
||||
Value: aws.String(targets[j]),
|
||||
})
|
||||
}
|
||||
}
|
||||
return changes
|
||||
}
|
||||
|
||||
type fields struct {
|
||||
changes []r53Types.Change
|
||||
maxSize int
|
||||
maxChars int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
want []batch
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "one_batch",
|
||||
fields: fields{
|
||||
changes: genChanges(r53Types.ChangeActionUpsert, r53Types.RRTypeA, "rec%04d", 99, "1.2.3.4"),
|
||||
maxSize: 1000,
|
||||
maxChars: 32000,
|
||||
},
|
||||
want: []batch{
|
||||
{start: 0, end: 99},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "multi_batch_size",
|
||||
fields: fields{
|
||||
changes: genChanges(r53Types.ChangeActionUpsert, r53Types.RRTypeA, "rec%04d", 2000, "1.2.3.4"),
|
||||
maxSize: 1000,
|
||||
maxChars: 32000,
|
||||
},
|
||||
want: []batch{
|
||||
{start: 0, end: 500},
|
||||
{start: 500, end: 1000},
|
||||
{start: 1000, end: 1500},
|
||||
{start: 1500, end: 2000},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "multi_batch_chars",
|
||||
fields: fields{
|
||||
changes: genChanges(r53Types.ChangeActionCreate, r53Types.RRTypeTxt, "rec%04d", 1000, "1.2.3.4", "1.2.3.5", "1.2.3.6", "1.2.3.7", "1.2.3.8", "1.2.3.9"),
|
||||
maxSize: 1000,
|
||||
maxChars: 32000,
|
||||
},
|
||||
want: []batch{
|
||||
{start: 0, end: 166},
|
||||
{start: 166, end: 332},
|
||||
{start: 332, end: 498},
|
||||
{start: 498, end: 664},
|
||||
{start: 664, end: 830},
|
||||
{start: 830, end: 996},
|
||||
{start: 996, end: 1000},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := &changeBatcher{
|
||||
changes: tt.fields.changes,
|
||||
maxSize: tt.fields.maxSize,
|
||||
maxChars: tt.fields.maxChars,
|
||||
}
|
||||
got := make([]batch, 0)
|
||||
for b.Next() {
|
||||
start, end := b.Batch()
|
||||
got = append(got, batch{
|
||||
start: start,
|
||||
end: end,
|
||||
})
|
||||
}
|
||||
err := b.Err()
|
||||
if tt.wantErr && err == nil {
|
||||
t.Errorf("%d: Expected an error, got nil", i)
|
||||
} else if !tt.wantErr && err != nil {
|
||||
t.Errorf("%d: Expected no error, got '%s'", i, err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("%d: Expected %s, got %s", i, tt.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue