From 801aae725b76df655e47848693736d294c03da4f Mon Sep 17 00:00:00 2001 From: Yuhui Xu Date: Fri, 30 Dec 2022 20:38:01 -0600 Subject: [PATCH] GCORE: Implement diff2 and greatly improve performance for getting record sets (#1867) Co-authored-by: Tom Limoncelli --- providers/gcore/convert.go | 4 +- providers/gcore/gcoreExtend.go | 105 +++++++++++++++++++++++++++++++ providers/gcore/gcoreProvider.go | 86 +++++++++++++++++-------- 3 files changed, 167 insertions(+), 28 deletions(-) create mode 100644 providers/gcore/gcoreExtend.go diff --git a/providers/gcore/convert.go b/providers/gcore/convert.go index 3c950ca96..e40a94d86 100644 --- a/providers/gcore/convert.go +++ b/providers/gcore/convert.go @@ -12,8 +12,10 @@ import ( ) // nativeToRecord takes a DNS record from G-Core and returns a native RecordConfig struct. -func nativeToRecords(n dnssdk.RRSet, zoneName string, recName string, recType string) ([]*models.RecordConfig, error) { +func nativeToRecords(n gcoreRRSetExtended, zoneName string) ([]*models.RecordConfig, error) { var rcs []*models.RecordConfig + recName := n.Name + recType := n.Type // Split G-Core's RRset into individual records for _, value := range n.Records { diff --git a/providers/gcore/gcoreExtend.go b/providers/gcore/gcoreExtend.go new file mode 100644 index 000000000..e40e11ead --- /dev/null +++ b/providers/gcore/gcoreExtend.go @@ -0,0 +1,105 @@ +package gcore + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "path" + "strings" + + dnssdk "github.com/G-Core/gcore-dns-sdk-go" +) + +type gcoreRRSets struct { + RRSets []gcoreRRSetExtended `json:"rrsets"` +} + +// Extended attributes over dnssdk.RRSet +type gcoreRRSetExtended struct { + Name string `json:"name"` + Type string `json:"type"` + + // Original + TTL int `json:"ttl"` + Records []dnssdk.ResourceRecord `json:"resource_records"` + Filters []dnssdk.RecordFilter `json:"filters"` +} + +func dnssdkDo(c *dnssdk.Client, apiKey string, ctx context.Context, method, uri string, bodyParams interface{}, dest interface{}) error { + // Adapted from https://github.com/G-Core/gcore-dns-sdk-go/blob/main/client.go#L289 + // No way to reflect a private method in Golang + + var bs []byte + if bodyParams != nil { + var err error + bs, err = json.Marshal(bodyParams) + if err != nil { + return fmt.Errorf("encode bodyParams: %w", err) + } + } + + endpoint, err := c.BaseURL.Parse(path.Join(c.BaseURL.Path, uri)) + if err != nil { + return fmt.Errorf("failed to parse endpoint: %w", err) + } + + if c.Debug { + log.Printf("[DEBUG] dns api request: %s %s %s \n", method, uri, bs) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), strings.NewReader(string(bs))) + if err != nil { + return fmt.Errorf("new request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("APIKey %s", apiKey)) + if c.UserAgent != "" { + req.Header.Set("User-Agent", c.UserAgent) + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("send request: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= http.StatusMultipleChoices { + all, _ := ioutil.ReadAll(resp.Body) + e := dnssdk.APIError{ + StatusCode: resp.StatusCode, + } + err := json.Unmarshal(all, &e) + if err != nil { + e.Message = string(all) + } + return e + } + + if dest == nil { + return nil + } + + // nolint: wrapcheck + return json.NewDecoder(resp.Body).Decode(dest) +} + +func (c *gcoreProvider) dnssdkRRSets(domain string) (gcoreRRSets, error) { + // Turns out G-Core has a hidden parameter "all=true" + // https://github.com/octodns/octodns-gcore/blob/main/octodns_gcore/__init__.py#L105 + // But this isn't exposed with their API, need to manually call it + + var result gcoreRRSets + url := fmt.Sprintf("/v2/zones/%s/rrsets?all=true", domain) + + err := dnssdkDo(c.provider, c.apiKey, c.ctx, http.MethodGet, url, nil, &result) + if err != nil { + return gcoreRRSets{}, err + } + + return result, nil +} diff --git a/providers/gcore/gcoreProvider.go b/providers/gcore/gcoreProvider.go index 546c043d3..28144d8e0 100644 --- a/providers/gcore/gcoreProvider.go +++ b/providers/gcore/gcoreProvider.go @@ -23,6 +23,7 @@ Info required in `creds.json`: type gcoreProvider struct { provider *dnssdk.Client ctx context.Context + apiKey string } // NewGCore creates the provider. @@ -34,6 +35,7 @@ func NewGCore(m map[string]string, metadata json.RawMessage) (providers.DNSServi c := &gcoreProvider{ provider: dnssdk.NewClient(dnssdk.PermanentAPIKeyAuth(m["api-key"])), ctx: context.TODO(), + apiKey: m["api-key"], } return c, nil @@ -94,14 +96,15 @@ func (c *gcoreProvider) GetZoneRecords(domain string) (models.Records, error) { // Convert RRsets to DNSControl format on the fly existingRecords := []*models.RecordConfig{} - // We cannot directly use Zone's ShortAnswers - // they aren't complete for CAA & SRV - for _, rec := range zone.Records { - rrset, err := c.provider.RRSet(c.ctx, zone.Name, rec.Name, rec.Type) - if err != nil { - return nil, err - } - nativeRecords, err := nativeToRecords(rrset, zone.Name, rec.Name, rec.Type) + // We cannot directly use Zone's ShortAnswers, they aren't complete for CAA & SRV + + rrsets, err := c.dnssdkRRSets(domain) + if err != nil { + return nil, err + } + + for _, rec := range rrsets.RRSets { + nativeRecords, err := nativeToRecords(rec, zone.Name) if err != nil { return nil, err } @@ -152,8 +155,11 @@ func generateChangeMsg(updates []string) string { // made. func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, existing models.Records) ([]*models.Correction, error) { + // Make delete happen earlier than creates & updates. var corrections []*models.Correction - if !diff2.EnableDiff2 || true { // Remove "|| true" when diff2 version arrives + var deletions []*models.Correction + + if !diff2.EnableDiff2 { // diff existing vs. current. differ := diff.New(dc) @@ -168,7 +174,6 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist desiredRecords := dc.Records.GroupedByKey() existingRecords := existing.GroupedByKey() - // First pass: delete records to avoid coexisting of conflicting types for label := range keysToUpdate { if _, ok := desiredRecords[label]; !ok { // record deleted in update @@ -177,20 +182,12 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist name := label.NameFQDN typ := label.Type msg := generateChangeMsg(keysToUpdate[label]) - corrections = append(corrections, &models.Correction{ + deletions = append(deletions, &models.Correction{ Msg: msg, F: func() error { return c.provider.DeleteRRSet(c.ctx, zone, name, typ) }, }) - } - } - - // Second pass: create and update records - for label := range keysToUpdate { - if _, ok := desiredRecords[label]; !ok { - // record deleted in update - // do nothing here } else if _, ok := existingRecords[label]; !ok { // record created in update @@ -203,12 +200,11 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist zone := dc.Name name := label.NameFQDN typ := label.Type - rec := *record msg := generateChangeMsg(keysToUpdate[label]) corrections = append(corrections, &models.Correction{ Msg: msg, F: func() error { - return c.provider.CreateRRSet(c.ctx, zone, name, typ, rec) + return c.provider.CreateRRSet(c.ctx, zone, name, typ, *record) }, }) @@ -223,21 +219,57 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist zone := dc.Name name := label.NameFQDN typ := label.Type - rec := *record msg := generateChangeMsg(keysToUpdate[label]) corrections = append(corrections, &models.Correction{ Msg: msg, F: func() error { - return c.provider.UpdateRRSet(c.ctx, zone, name, typ, rec) + return c.provider.UpdateRRSet(c.ctx, zone, name, typ, *record) }, }) } } - return corrections, nil + } else { + // Diff2 version + changes, err := diff2.ByRecordSet(existing, dc, nil) + if err != nil { + return nil, err + } + + for _, change := range changes { + record := recordsToNative(change.New, change.Key) + + // Copy all params to avoid overwrites + zone := dc.Name + name := change.Key.NameFQDN + typ := change.Key.Type + msg := generateChangeMsg(change.Msgs) + + switch change.Type { + case diff2.CREATE: + corrections = append(corrections, &models.Correction{ + Msg: msg, + F: func() error { + return c.provider.CreateRRSet(c.ctx, zone, name, typ, *record) + }, + }) + case diff2.CHANGE: + corrections = append(corrections, &models.Correction{ + Msg: msg, + F: func() error { + return c.provider.UpdateRRSet(c.ctx, zone, name, typ, *record) + }, + }) + case diff2.DELETE: + deletions = append(deletions, &models.Correction{ + Msg: msg, + F: func() error { + return c.provider.DeleteRRSet(c.ctx, zone, name, typ) + }, + }) + } + } } - // Insert Future diff2 version here. - - return corrections, nil + return append(deletions, corrections...), nil }