GCORE: Implement diff2 and greatly improve performance for getting record sets (#1867)

Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>

providers/gcore/convert.go

@@ -12,8 +12,10 @@ import (
 )
 
 // nativeToRecord takes a DNS record from G-Core and returns a native RecordConfig struct.
-func nativeToRecords(n dnssdk.RRSet, zoneName string, recName string, recType string) ([]*models.RecordConfig, error) {
+func nativeToRecords(n gcoreRRSetExtended, zoneName string) ([]*models.RecordConfig, error) {
 	var rcs []*models.RecordConfig
+	recName := n.Name
+	recType := n.Type
 
 	// Split G-Core's RRset into individual records
 	for _, value := range n.Records {

providers/gcore/gcoreExtend.go

@@ -0,0 +1,105 @@
+package gcore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"path"
+	"strings"
+
+	dnssdk "github.com/G-Core/gcore-dns-sdk-go"
+)
+
+type gcoreRRSets struct {
+	RRSets []gcoreRRSetExtended `json:"rrsets"`
+}
+
+// Extended attributes over dnssdk.RRSet
+type gcoreRRSetExtended struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+
+	// Original
+	TTL     int                     `json:"ttl"`
+	Records []dnssdk.ResourceRecord `json:"resource_records"`
+	Filters []dnssdk.RecordFilter   `json:"filters"`
+}
+
+func dnssdkDo(c *dnssdk.Client, apiKey string, ctx context.Context, method, uri string, bodyParams interface{}, dest interface{}) error {
+	// Adapted from https://github.com/G-Core/gcore-dns-sdk-go/blob/main/client.go#L289
+	// No way to reflect a private method in Golang
+
+	var bs []byte
+	if bodyParams != nil {
+		var err error
+		bs, err = json.Marshal(bodyParams)
+		if err != nil {
+			return fmt.Errorf("encode bodyParams: %w", err)
+		}
+	}
+
+	endpoint, err := c.BaseURL.Parse(path.Join(c.BaseURL.Path, uri))
+	if err != nil {
+		return fmt.Errorf("failed to parse endpoint: %w", err)
+	}
+
+	if c.Debug {
+		log.Printf("[DEBUG] dns api request: %s %s %s \n", method, uri, bs)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), strings.NewReader(string(bs)))
+	if err != nil {
+		return fmt.Errorf("new request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("APIKey %s", apiKey))
+	if c.UserAgent != "" {
+		req.Header.Set("User-Agent", c.UserAgent)
+	}
+
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("send request: %w", err)
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode >= http.StatusMultipleChoices {
+		all, _ := ioutil.ReadAll(resp.Body)
+		e := dnssdk.APIError{
+			StatusCode: resp.StatusCode,
+		}
+		err := json.Unmarshal(all, &e)
+		if err != nil {
+			e.Message = string(all)
+		}
+		return e
+	}
+
+	if dest == nil {
+		return nil
+	}
+
+	// nolint: wrapcheck
+	return json.NewDecoder(resp.Body).Decode(dest)
+}
+
+func (c *gcoreProvider) dnssdkRRSets(domain string) (gcoreRRSets, error) {
+	// Turns out G-Core has a hidden parameter "all=true"
+	// https://github.com/octodns/octodns-gcore/blob/main/octodns_gcore/__init__.py#L105
+	// But this isn't exposed with their API, need to manually call it
+
+	var result gcoreRRSets
+	url := fmt.Sprintf("/v2/zones/%s/rrsets?all=true", domain)
+
+	err := dnssdkDo(c.provider, c.apiKey, c.ctx, http.MethodGet, url, nil, &result)
+	if err != nil {
+		return gcoreRRSets{}, err
+	}
+
+	return result, nil
+}

providers/gcore/gcoreProvider.go

@@ -23,6 +23,7 @@ Info required in `creds.json`:
 type gcoreProvider struct {
 	provider *dnssdk.Client
 	ctx      context.Context
+	apiKey   string
 }
 
 // NewGCore creates the provider.
@@ -34,6 +35,7 @@ func NewGCore(m map[string]string, metadata json.RawMessage) (providers.DNSServi
 	c := &gcoreProvider{
 		provider: dnssdk.NewClient(dnssdk.PermanentAPIKeyAuth(m["api-key"])),
 		ctx:      context.TODO(),
+		apiKey:   m["api-key"],
 	}
 
 	return c, nil
@@ -94,14 +96,15 @@ func (c *gcoreProvider) GetZoneRecords(domain string) (models.Records, error) {
 	// Convert RRsets to DNSControl format on the fly
 	existingRecords := []*models.RecordConfig{}
 
-	// We cannot directly use Zone's ShortAnswers
-	// they aren't complete for CAA & SRV
-	for _, rec := range zone.Records {
-		rrset, err := c.provider.RRSet(c.ctx, zone.Name, rec.Name, rec.Type)
-		if err != nil {
-			return nil, err
-		}
-		nativeRecords, err := nativeToRecords(rrset, zone.Name, rec.Name, rec.Type)
+	// We cannot directly use Zone's ShortAnswers, they aren't complete for CAA & SRV
+
+	rrsets, err := c.dnssdkRRSets(domain)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, rec := range rrsets.RRSets {
+		nativeRecords, err := nativeToRecords(rec, zone.Name)
 		if err != nil {
 			return nil, err
 		}
@@ -152,8 +155,11 @@ func generateChangeMsg(updates []string) string {
 // made.
 func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, existing models.Records) ([]*models.Correction, error) {
 
+	// Make delete happen earlier than creates & updates.
 	var corrections []*models.Correction
-	if !diff2.EnableDiff2 || true { // Remove "|| true" when diff2 version arrives
+	var deletions []*models.Correction
+
+	if !diff2.EnableDiff2 {
 
 		// diff existing vs. current.
 		differ := diff.New(dc)
@@ -168,7 +174,6 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist
 		desiredRecords := dc.Records.GroupedByKey()
 		existingRecords := existing.GroupedByKey()
 
-		// First pass: delete records to avoid coexisting of conflicting types
 		for label := range keysToUpdate {
 			if _, ok := desiredRecords[label]; !ok {
 				// record deleted in update
@@ -177,20 +182,12 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist
 				name := label.NameFQDN
 				typ := label.Type
 				msg := generateChangeMsg(keysToUpdate[label])
-				corrections = append(corrections, &models.Correction{
+				deletions = append(deletions, &models.Correction{
 					Msg: msg,
 					F: func() error {
 						return c.provider.DeleteRRSet(c.ctx, zone, name, typ)
 					},
 				})
-			}
-		}
-
-		// Second pass: create and update records
-		for label := range keysToUpdate {
-			if _, ok := desiredRecords[label]; !ok {
-				// record deleted in update
-				// do nothing here
 
 			} else if _, ok := existingRecords[label]; !ok {
 				// record created in update
@@ -203,12 +200,11 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist
 				zone := dc.Name
 				name := label.NameFQDN
 				typ := label.Type
-				rec := *record
 				msg := generateChangeMsg(keysToUpdate[label])
 				corrections = append(corrections, &models.Correction{
 					Msg: msg,
 					F: func() error {
-						return c.provider.CreateRRSet(c.ctx, zone, name, typ, rec)
+						return c.provider.CreateRRSet(c.ctx, zone, name, typ, *record)
 					},
 				})
 
@@ -223,21 +219,57 @@ func (c *gcoreProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist
 				zone := dc.Name
 				name := label.NameFQDN
 				typ := label.Type
-				rec := *record
 				msg := generateChangeMsg(keysToUpdate[label])
 				corrections = append(corrections, &models.Correction{
 					Msg: msg,
 					F: func() error {
-						return c.provider.UpdateRRSet(c.ctx, zone, name, typ, rec)
+						return c.provider.UpdateRRSet(c.ctx, zone, name, typ, *record)
 					},
 				})
 			}
 		}
 
-		return corrections, nil
+	} else {
+		// Diff2 version
+		changes, err := diff2.ByRecordSet(existing, dc, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, change := range changes {
+			record := recordsToNative(change.New, change.Key)
+
+			// Copy all params to avoid overwrites
+			zone := dc.Name
+			name := change.Key.NameFQDN
+			typ := change.Key.Type
+			msg := generateChangeMsg(change.Msgs)
+
+			switch change.Type {
+			case diff2.CREATE:
+				corrections = append(corrections, &models.Correction{
+					Msg: msg,
+					F: func() error {
+						return c.provider.CreateRRSet(c.ctx, zone, name, typ, *record)
+					},
+				})
+			case diff2.CHANGE:
+				corrections = append(corrections, &models.Correction{
+					Msg: msg,
+					F: func() error {
+						return c.provider.UpdateRRSet(c.ctx, zone, name, typ, *record)
+					},
+				})
+			case diff2.DELETE:
+				deletions = append(deletions, &models.Correction{
+					Msg: msg,
+					F: func() error {
+						return c.provider.DeleteRRSet(c.ctx, zone, name, typ)
+					},
+				})
+			}
+		}
 	}
 
-	// Insert Future diff2 version here.
-
-	return corrections, nil
+	return append(deletions, corrections...), nil
 }