HUAWEICLOUD: add metadata to control Intelligent Resolution (#3013)

2024-09-20 06:46:19 +08:00 · 2024-06-19 04:17:02 +08:00 · 2024-06-19 04:17:02 +08:00 · 2d15884eb3
parent ce07c76fe8
commit 2d15884eb3
6 changed files with 226 additions and 49 deletions
--- a/.github/workflows/pr_test.yml
+++ b/.github/workflows/pr_test.yml
@ -117,6 +117,7 @@ jobs:
      GCLOUD_DOMAIN: ${{ vars.GCLOUD_DOMAIN }}
      HEDNS_DOMAIN: ${{ vars.HEDNS_DOMAIN }}
      HEXONET_DOMAIN: ${{ vars.HEXONET_DOMAIN }}
+      HUAWEICLOUD_DOMAIN: ${{ vars.HUAWEICLOUD_DOMAIN }}
      NAMEDOTCOM_DOMAIN: ${{ vars.NAMEDOTCOM_DOMAIN }}
      NS1_DOMAIN: ${{ vars.NS1_DOMAIN }}
      POWERDNS_DOMAIN: ${{ vars.POWERDNS_DOMAIN }}
@ -161,6 +162,10 @@ jobs:
      HEXONET_PW: ${{ secrets.HEXONET_PW }}
      HEXONET_UID: ${{ secrets.HEXONET_UID }}

+      HUAWEICLOUD_REGION: ${{ secrets.HUAWEICLOUD_REGION }}
+      HUAWEICLOUD_KEY_ID: ${{ secrets.HUAWEICLOUD_KEY_ID }}
+      HUAWEICLOUD_KEY: ${{ secrets.HUAWEICLOUD_KEY }}
+
      NAMEDOTCOM_KEY: ${{ secrets.NAMEDOTCOM_KEY }}
      NAMEDOTCOM_URL: ${{ secrets.NAMEDOTCOM_URL }}
      NAMEDOTCOM_USER: ${{ secrets.NAMEDOTCOM_USER }}
--- a/documentation/provider/huaweicloud.md
+++ b/documentation/provider/huaweicloud.md
@ -20,7 +20,37 @@ Example:
 {% endcode %}

 ## Metadata
-This provider does not recognize any special metadata fields unique to Huawei Cloud DNS.
+There are some record level metadata available for this provider:
+   * `hw_line` (Line ID, default "default_view") Refer to the [Intelligent Resolution](https://support.huaweicloud.com/intl/en-us/usermanual-dns/dns_usermanual_0041.html) for more information.
+     * Available Line ID refer to [Resolution Lines](https://support.huaweicloud.com/intl/en-us/api-dns/en-us_topic_0085546214.html). Custom Line ID can also be used.
+   * `hw_weight` (0-1000, default "1") Refer to the [Configuring Weighted Routing](https://support.huaweicloud.com/intl/en-us/usermanual-dns/dns_usermanual_0705.html) for more information.
+   * `hw_rrset_key` (default "") User defined key for RRset load balance. This value would be stored in the description field of the RRset.
+
+The following example shows how to use the metadata:
+
+{% code title="dnsconfig.js" %}
+```javascript
+var REG_NONE = NewRegistrar("none");
+var DSP_HWCLOUD = NewDnsProvider("huaweicloud");
+
+D("example.com", REG_NONE, DnsProvider(DSP_HWCLOUD),
+    // this example will create 4 rrsets with the same name "test"
+    A("test", "8.8.8.8"),
+    A("test", "8.8.4.4"),
+    A("test", "9.9.9.9", {hw_weight: "10"}),         // Weighted Routing
+    A("test", "149.112.112.112", {hw_weight: "10"}), // Weighted Routing
+    A("test", "223.5.5.5", {hw_line: "CN"}), // GEODNS
+    A("test", "223.6.6.6", {hw_line: "CN", hw_weight: "10"}), // GEODNS with weight
+
+    // this example will create 3 rrsets with the same name "lb"
+    A("rr-lb", "10.0.0.1", {hw_weight: "10", hw_rrset_key: "lb-zone-a"}),
+    A("rr-lb", "10.0.0.2", {hw_weight: "10", hw_rrset_key: "lb-zone-a"}),
+    A("rr-lb", "10.0.1.1", {hw_weight: "10", hw_rrset_key: "lb-zone-b"}),
+    A("rr-lb", "10.0.1.2", {hw_weight: "10", hw_rrset_key: "lb-zone-b"}),
+    A("rr-lb", "10.0.2.2", {hw_weight: "0",  hw_rrset_key: "lb-zone-c"}),
+END);
+```
+{% endcode %}

 ## Usage
 An example configuration:
@ -71,7 +101,3 @@ If that doesn't work, log into Huaweicloud's website and open the [API Explorer]

 ## New domains
 If a domain does not exist in your Huawei Cloud account, DNSControl will automatically add it with the `push` command.
-
-## GeoDNS
-Managing GeoDNS RRSet on Huawei Cloud (also called **Line** in Huawei Cloud DNS) is not supported in DNSControl.
-If your Zone needs to use GeoDNS, please create it manually in the console and use [IGNORE](../language-reference/domain-modifiers/IGNORE.md) modifiers in DNSControl to prevent changing it.
--- a/integrationTest/providers.json
+++ b/integrationTest/providers.json
@ -154,6 +154,13 @@
    "authToken": "$HOSTINGDE_AUTHTOKEN",
    "domain": "$HOSTINGDE_DOMAIN"
  },
+  "HUAWEICLOUD": {
+    "TYPE": "HUAWEICLOUD",
+    "domain": "$HUAWEICLOUD_DOMAIN",
+    "Region": "$HUAWEICLOUD_REGION",
+    "KeyId": "$HUAWEICLOUD_KEY_ID",
+    "SecretKey": "$HUAWEICLOUD_KEY"
+  },
  "INWX": {
    "TYPE": "INWX",
    "domain": "$INWX_DOMAIN",
@ -288,12 +295,5 @@
    "TYPE": "VULTR",
    "domain": "$VULTR_DOMAIN",
    "token": "$VULTR_TOKEN"
-  },
-  "HUAWEICLOUD": {
-    "TYPE": "HUAWEICLOUD",
-    "domain": "$HUAWEICLOUD_DOMAIN",
-    "Region": "$HUAWEICLOUD_REGION",
-    "KeyId": "$HUAWEICLOUD_KEY_ID",
-    "SecretKey": "$HUAWEICLOUD_KEY"
  }
 }
--- a/providers/huaweicloud/convert.go
+++ b/providers/huaweicloud/convert.go
@ -3,6 +3,7 @@ package huaweicloud
 import (
 	"fmt"
 	"slices"
+	"strconv"

 	"github.com/StackExchange/dnscontrol/v4/models"
 	"github.com/StackExchange/dnscontrol/v4/pkg/printer"
@ -15,16 +16,16 @@ func getRRSetIDFromRecords(rcs models.Records) []string {
 		if r.Original == nil {
 			continue
 		}
-		if r.Original.(*model.ListRecordSets).Id == nil {
+		if r.Original.(*model.ShowRecordSetByZoneResp).Id == nil {
 			printer.Warnf("RecordSet ID is nil for record %+v\n", r)
 			continue
 		}
-		ids = append(ids, *r.Original.(*model.ListRecordSets).Id)
+		ids = append(ids, *r.Original.(*model.ShowRecordSetByZoneResp).Id)
 	}
 	return slices.Compact(ids)
 }

-func nativeToRecords(n *model.ListRecordSets, zoneName string) (models.Records, error) {
+func nativeToRecords(n *model.ShowRecordSetByZoneResp, zoneName string) (models.Records, error) {
 	if n.Name == nil || n.Type == nil || n.Records == nil || n.Ttl == nil {
 		return nil, fmt.Errorf("missing required fields in Huaweicloud's RRset: %+v", n)
 	}
@ -37,26 +38,71 @@ func nativeToRecords(n *model.ListRecordSets, zoneName string) (models.Records,
 		rc := &models.RecordConfig{
 			TTL:      uint32(*n.Ttl),
 			Original: n,
+			Metadata: map[string]string{},
 		}
 		rc.SetLabelFromFQDN(recName, zoneName)
 		if err := rc.PopulateFromString(recType, value, zoneName); err != nil {
 			return nil, fmt.Errorf("unparsable record received from Huaweicloud: %w", err)
 		}
+		if n.Line != nil {
+			rc.Metadata[metaLine] = *n.Line
+		}
+		if n.Weight != nil {
+			rc.Metadata[metaWeight] = fmt.Sprintf("%d", *n.Weight)
+		}
+		if n.Description != nil {
+			rc.Metadata[metaKey] = *n.Description
+		}
 		rcs = append(rcs, rc)
 	}

 	return rcs, nil
 }

-func recordsToNative(rcs models.Records, expectedKey models.RecordKey) *model.ListRecordSets {
+func recordsToNative(rcs models.Records, expectedKey models.RecordKey) (*model.ShowRecordSetByZoneResp, error) {
+	// rcs length is guaranteed to be > 0
+	if len(rcs) == 0 {
+		return nil, fmt.Errorf("empty record set")
+	}
+	// line and weight should be the same for all records in the rrset
+	line := rcs[0].Metadata[metaLine]
+	weightStr := rcs[0].Metadata[metaWeight]
+	for _, r := range rcs {
+		if r.Metadata[metaLine] != line {
+			return nil, fmt.Errorf("all records in the rrset must have the same line %s", line)
+		}
+		if r.Metadata[metaWeight] != weightStr {
+			return nil, fmt.Errorf("all records in the rrset must have the same weight %s", weightStr)
+		}
+	}
+
+	// parse weight to int32
+	var weight *int32
+	if weightStr != "" {
+		weightInt, err := strconv.ParseInt(weightStr, 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse weight %s to int32", weightStr)
+		}
+		weightInt32 := int32(weightInt)
+		// weight should be 0-1000
+		if weightInt32 < 0 || weightInt32 > 1000 {
+			return nil, fmt.Errorf("weight must be between 0 and 1000")
+		}
+		weight = &weightInt32
+	}
+
 	resultTTL := int32(0)
 	resultVal := []string{}
 	name := expectedKey.NameFQDN + "."
-	result := &model.ListRecordSets{
+	key := rcs[0].Metadata[metaKey]
+	result := &model.ShowRecordSetByZoneResp{
 		Name:    &name,
 		Type:    &expectedKey.Type,
 		Ttl:     &resultTTL,
 		Records: &resultVal,
+		Line:    &line,
+		Weight:  weight,
+		Description: &key,
 	}

 	for _, r := range rcs {
@ -84,5 +130,5 @@ func recordsToNative(rcs models.Records, expectedKey models.RecordKey) *model.Li
 		}
 	}

-	return result
+	return result, nil
 }
--- a/providers/huaweicloud/huaweicloudProvider.go
+++ b/providers/huaweicloud/huaweicloudProvider.go
@ -20,10 +20,18 @@ import (

 /*
 Huaweicloud API DNS provider:
+
 Info required in `creds.json`:
   - KeyId
   - SecretKey
   - Region
+
+Record level metadata available:
+   - hw_line (refer below Huawei Cloud DNS API documentation for available lines, default "default_view")
+	 					 (https://support.huaweicloud.com/intl/en-us/api-dns/en-us_topic_0085546214.html)
+   - hw_weight (0-1000, default "1")
+	 - hw_rrset_key (default "")
+
 */

 type huaweicloudProvider struct {
@ -33,6 +41,14 @@ type huaweicloudProvider struct {
 	region         *region.Region
 }

+const (
+	metaWeight    = "hw_weight"
+	metaLine      = "hw_line"
+	metaKey       = "hw_rrset_key"
+	defaultWeight = "1"
+	defaultLine   = "default_view"
+)
+
 // newHuaweicloud creates the provider.
 func newHuaweicloud(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
 	auth, err := basic.NewCredentialsBuilder().
--- a/providers/huaweicloud/records.go
+++ b/providers/huaweicloud/records.go
@ -54,12 +54,14 @@ func (c *huaweicloudProvider) GetZoneRecordsCorrections(dc *models.DomainConfig,
 		return nil, fmt.Errorf("zone %s not found", dc.Name)
 	}

+	addDefaultMeta(dc.Records)
+
 	// Make delete happen earlier than creates & updates.
 	var corrections []*models.Correction
 	var deletions []*models.Correction
 	var reports []*models.Correction

-	changes, err := diff2.ByRecordSet(existing, dc, nil)
+	changes, err := diff2.ByRecordSet(existing, dc, genComparable)
 	if err != nil {
 		return nil, err
 	}
@ -71,20 +73,49 @@ func (c *huaweicloudProvider) GetZoneRecordsCorrections(dc *models.DomainConfig,
 		case diff2.CREATE:
 			fallthrough
 		case diff2.CHANGE:
-			records := recordsToNative(change.New, change.Key)
-			rrsetsID := getRRSetIDFromRecords(change.Old)
+			newRecordsColl := collectRecordsByLineAndWeightAndKey(change.New)
+			oldRecordsColl := collectRecordsByLineAndWeightAndKey(change.Old)
 			corrections = append(corrections, &models.Correction{
 				Msg: change.MsgsJoined,
 				F: func() error {
-					if len(rrsetsID) == 1 {
-						return c.updateRRSet(zoneID, rrsetsID[0], records)
-					} else {
-						err := c.deleteRRSets(zoneID, rrsetsID)
+					// delete old records if not exist in new records
+					for key, oldRecords := range oldRecordsColl {
+						if _, ok := newRecordsColl[key]; !ok {
+							rrsetIDOld := getRRSetIDFromRecords(oldRecords)
+							err := c.deleteRRSets(zoneID, rrsetIDOld)
 							if err != nil {
 								return err
 							}
-						return c.createRRSet(zoneID, records)
 						}
+					}
+					// modify or create new records
+					for key, newRecords := range newRecordsColl {
+						records, err := recordsToNative(newRecords, change.Key)
+						if err != nil {
+							return err
+						}
+						oldRecords := oldRecordsColl[key]
+						rrsetIDOld := getRRSetIDFromRecords(oldRecords)
+
+						if len(rrsetIDOld) == 1 {
+							// update existing rrset
+							err = c.updateRRSet(zoneID, rrsetIDOld[0], records)
+							if err != nil {
+								return err
+							}
+						} else {
+							// create new rrset or combine multiple rrsets into one
+							err := c.deleteRRSets(zoneID, rrsetIDOld)
+							if err != nil {
+								return err
+							}
+							err = c.createRRSet(zoneID, records)
+							if err != nil {
+								return err
+							}
+						}
+					}
+					return nil
 				},
 			})
 		case diff2.DELETE:
@ -105,15 +136,63 @@ func (c *huaweicloudProvider) GetZoneRecordsCorrections(dc *models.DomainConfig,
 	return result, nil
 }

+func collectRecordsByLineAndWeightAndKey(records models.Records) map[string]models.Records {
+	recordsByLineAndWeight := make(map[string]models.Records)
+	for _, rec := range records {
+		line := rec.Metadata[metaLine]
+		weight := rec.Metadata[metaWeight]
+		rrsetKey := rec.Metadata[metaKey]
+		key := weight + "," + line + "," + rrsetKey
+		if _, ok := recordsByLineAndWeight[key]; !ok {
+			recordsByLineAndWeight[key] = models.Records{}
+		}
+		recordsByLineAndWeight[key] = append(recordsByLineAndWeight[key], rec)
+	}
+	return recordsByLineAndWeight
+}
+
+func addDefaultMeta(recs models.Records) {
+	for _, r := range recs {
+		if r.Metadata == nil {
+			r.Metadata = make(map[string]string)
+		}
+		if r.Metadata[metaLine] == "" {
+			r.Metadata[metaLine] = defaultLine
+		}
+		// apex ns should not have weight
+		isApexNS := r.Type == "NS" && r.Name == "@"
+		if !isApexNS && r.Metadata[metaWeight] == "" {
+			r.Metadata[metaWeight] = defaultWeight
+		}
+	}
+}
+
+func genComparable(rec *models.RecordConfig) string {
+	// apex ns
+	if rec.Type == "NS" && rec.Name == "@" {
+		return ""
+	}
+	weight := rec.Metadata[metaWeight]
+	line := rec.Metadata[metaLine]
+	key := rec.Metadata[metaKey]
+	if weight == "" {
+		weight = defaultWeight
+	}
+	if line == "" {
+		line = defaultLine
+	}
+	return "weight=" + weight + " line=" + line + " key=" + key
+}
+
 func (c *huaweicloudProvider) deleteRRSets(zoneID string, rrsets []string) error {
 	for _, rrset := range rrsets {
-		deletePayload := &model.DeleteRecordSetRequest{
+		deletePayload := &model.DeleteRecordSetsRequest{
 			ZoneId:      zoneID,
 			RecordsetId: rrset,
 		}
 		var err error
 		withRetry(func() error {
-			_, err = c.client.DeleteRecordSet(deletePayload)
+			_, err = c.client.DeleteRecordSets(deletePayload)
 			return err
 		})
 		if err != nil {
@ -123,41 +202,46 @@ func (c *huaweicloudProvider) deleteRRSets(zoneID string, rrsets []string) error
 	return nil
 }

-func (c *huaweicloudProvider) createRRSet(zoneID string, rc *model.ListRecordSets) error {
-	createPayload := &model.CreateRecordSetRequest{
+func (c *huaweicloudProvider) createRRSet(zoneID string, rc *model.ShowRecordSetByZoneResp) error {
+	createPayload := &model.CreateRecordSetWithLineRequest{
 		ZoneId: zoneID,
-		Body: &model.CreateRecordSetRequestBody{
-			Name:    *rc.Name,
-			Type:    *rc.Type,
-			Ttl:     rc.Ttl,
-			Records: *rc.Records,
-		},
-	}
-	var err error
-	withRetry(func() error {
-		_, err = c.client.CreateRecordSet(createPayload)
-		return err
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (c *huaweicloudProvider) updateRRSet(zoneID, rrsetID string, rc *model.ListRecordSets) error {
-	updatePayload := &model.UpdateRecordSetRequest{
-		ZoneId:      zoneID,
-		RecordsetId: rrsetID,
-		Body: &model.UpdateRecordSetReq{
+		Body: &model.CreateRecordSetWithLineRequestBody{
 			Name:        *rc.Name,
 			Type:        *rc.Type,
 			Ttl:         rc.Ttl,
 			Records:     rc.Records,
+			Weight:      rc.Weight,
+			Line:        rc.Line,
+			Description: rc.Description,
 		},
 	}
 	var err error
 	withRetry(func() error {
-		_, err = c.client.UpdateRecordSet(updatePayload)
+		_, err = c.client.CreateRecordSetWithLine(createPayload)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *huaweicloudProvider) updateRRSet(zoneID, rrsetID string, rc *model.ShowRecordSetByZoneResp) error {
+	updatePayload := &model.UpdateRecordSetsRequest{
+		ZoneId:      zoneID,
+		RecordsetId: rrsetID,
+		Body: &model.UpdateRecordSetsReq{
+			Name:        *rc.Name,
+			Type:        *rc.Type,
+			Ttl:         rc.Ttl,
+			Records:     rc.Records,
+			Weight:      rc.Weight,
+			Description: rc.Description,
+		},
+	}
+	var err error
+	withRetry(func() error {
+		_, err = c.client.UpdateRecordSets(updatePayload)
 		return err
 	})
 	if err != nil {
@ -180,20 +264,20 @@ func parseMarkerFromURL(link string) (string, error) {
 	return marker, nil
 }

-func (c *huaweicloudProvider) fetchZoneRecordsFromRemote(zoneID string) (*[]model.ListRecordSets, error) {
+func (c *huaweicloudProvider) fetchZoneRecordsFromRemote(zoneID string) (*[]model.ShowRecordSetByZoneResp, error) {
 	var nextMarker *string
-	existingRecords := []model.ListRecordSets{}
+	existingRecords := []model.ShowRecordSetByZoneResp{}
 	availableStatus := []string{"ACTIVE", "PENDING_CREATE", "PENDING_UPDATE"}

 	for {
-		payload := model.ListRecordSetsByZoneRequest{
+		payload := model.ShowRecordSetByZoneRequest{
 			ZoneId: zoneID,
 			Marker: nextMarker,
 		}
-		var res *model.ListRecordSetsByZoneResponse
+		var res *model.ShowRecordSetByZoneResponse
 		var err error
 		withRetry(func() error {
-			res, err = c.client.ListRecordSetsByZone(&payload)
+			res, err = c.client.ShowRecordSetByZone(&payload)
 			return err
 		})
 		if err != nil {