From 54fc2e9ce334ff6cee0012f9a162ed36df0f7c56 Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Sun, 11 Dec 2022 17:28:58 -0500 Subject: [PATCH] NEW FEATURE: diff2: A better "diff" mechanism (#1852) --- go.mod | 2 +- models/domain.go | 26 +- models/record.go | 11 +- package-lock.json | 30 +- pkg/diff/diff.go | 2 +- pkg/diff2/analyze.go | 275 ++++++++ pkg/diff2/analyze_test.go | 635 ++++++++++++++++++ pkg/diff2/compareconfig.go | 214 ++++++ pkg/diff2/compareconfig_test.go | 214 ++++++ pkg/diff2/diff2.go | 185 +++++ pkg/diff2/groupsort.go | 45 ++ pkg/diff2/groupsort_test.go | 45 ++ pkg/diff2/highest.go | 7 + pkg/diff2/min.go | 10 + pkg/diff2/nopurge.go | 22 + pkg/diff2/notes.txt | 198 ++++++ pkg/diff2/unmanaged.go | 129 ++++ pkg/diff2/unmanaged_test.go | 147 ++++ pkg/diff2/verb_string.go | 26 + pkg/js/helpers.js | 40 +- pkg/js/js_test.go | 2 +- pkg/js/parse_tests/005-ignored-records.json | 54 +- .../parse_tests/023-ignored-glob-records.json | 9 +- pkg/js/parse_tests/042-unmanaged.js | 6 + pkg/js/parse_tests/042-unmanaged.json | 34 + pkg/js/parse_tests/043-safety.js | 5 + pkg/js/parse_tests/043-safety.json | 19 + pkg/prettyzone/prettyzone_test.go | 4 +- pkg/prettyzone/sorting.go | 6 +- providers/autodns/autoDnsProvider.go | 1 + providers/axfrddns/axfrddnsProvider.go | 1 + providers/azuredns/azureDnsProvider.go | 89 ++- providers/bind/bindProvider.go | 80 ++- providers/cloudflare/cloudflareProvider.go | 1 + providers/cloudns/cloudnsProvider.go | 1 + providers/cscglobal/dns.go | 1 + providers/desec/desecProvider.go | 1 + providers/gandiv5/gandi_v5Provider.go | 81 ++- providers/oracle/oracleProvider.go | 2 - providers/ovh/ovhProvider.go | 2 - 40 files changed, 2581 insertions(+), 81 deletions(-) create mode 100644 pkg/diff2/analyze.go create mode 100644 pkg/diff2/analyze_test.go create mode 100644 pkg/diff2/compareconfig.go create mode 100644 pkg/diff2/compareconfig_test.go create mode 100644 pkg/diff2/diff2.go create mode 100644 pkg/diff2/groupsort.go create mode 100644 pkg/diff2/groupsort_test.go create mode 100644 pkg/diff2/highest.go create mode 100644 pkg/diff2/min.go create mode 100644 pkg/diff2/nopurge.go create mode 100644 pkg/diff2/notes.txt create mode 100644 pkg/diff2/unmanaged.go create mode 100644 pkg/diff2/unmanaged_test.go create mode 100644 pkg/diff2/verb_string.go create mode 100644 pkg/js/parse_tests/042-unmanaged.js create mode 100644 pkg/js/parse_tests/042-unmanaged.json create mode 100644 pkg/js/parse_tests/043-safety.js create mode 100644 pkg/js/parse_tests/043-safety.json diff --git a/go.mod b/go.mod index 5f9a07dc3..1d38c7e72 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( require ( github.com/G-Core/gcore-dns-sdk-go v0.2.3 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-isatty v0.0.16 github.com/vultr/govultr/v2 v2.17.2 golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 @@ -129,7 +130,6 @@ require ( github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect github.com/juju/testing v0.0.0-20210324180055-18c50b0c2098 // indirect github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b // indirect - github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/models/domain.go b/models/domain.go index e9f5d32b4..8c1b4cd7a 100644 --- a/models/domain.go +++ b/models/domain.go @@ -15,13 +15,17 @@ type DomainConfig struct { RegistrarName string `json:"registrar"` DNSProviderNames map[string]int `json:"dnsProviders"` - Metadata map[string]string `json:"meta,omitempty"` - Records Records `json:"records"` - Nameservers []*Nameserver `json:"nameservers,omitempty"` - KeepUnknown bool `json:"keepunknown,omitempty"` - IgnoredNames []*IgnoreName `json:"ignored_names,omitempty"` - IgnoredTargets []*IgnoreTarget `json:"ignored_targets,omitempty"` - AutoDNSSEC string `json:"auto_dnssec,omitempty"` // "", "on", "off" + Metadata map[string]string `json:"meta,omitempty"` + Records Records `json:"records"` + Nameservers []*Nameserver `json:"nameservers,omitempty"` + + KeepUnknown bool `json:"keepunknown,omitempty"` + IgnoredNames []*IgnoreName `json:"ignored_names,omitempty"` + IgnoredTargets []*IgnoreTarget `json:"ignored_targets,omitempty"` + Unmanaged []*UnmanagedConfig `json:"unmanaged,omitempty"` + UnmanagedUnsafe bool `json:"unmanaged_disable_safety_check,omitempty"` + + AutoDNSSEC string `json:"auto_dnssec,omitempty"` // "", "on", "off" //DNSSEC bool `json:"dnssec,omitempty"` // These fields contain instantiated provider instances once everything is linked up. @@ -32,6 +36,14 @@ type DomainConfig struct { DNSProviderInstances []*DNSProviderInstance `json:"-"` } +// UnmanagedConfig describes an UNMANAGED() rule. +type UnmanagedConfig struct { + Label string `json:"label_pattern"` // Glob pattern for matching labels. + RType string `json:"rType_pattern"` // Comma-separated list of DNS Resource Types. + typeMap map[string]bool // map of RTypes or len()=0 for all + Target string `json:"target_pattern"` // Glob pattern for matching targets. +} + // Copy returns a deep copy of the DomainConfig. func (dc *DomainConfig) Copy() (*DomainConfig, error) { newDc := &DomainConfig{} diff --git a/models/record.go b/models/record.go index 47fb33f1e..e89700701 100644 --- a/models/record.go +++ b/models/record.go @@ -305,10 +305,13 @@ func (rc *RecordConfig) GetLabelFQDN() string { // ToDiffable returns a string that is comparable by a differ. // extraMaps: a list of maps that should be included in the comparison. func (rc *RecordConfig) ToDiffable(extraMaps ...map[string]string) string { - content := fmt.Sprintf("%v ttl=%d", rc.GetTargetCombined(), rc.TTL) - if rc.Type == "SOA" { + var content string + switch rc.Type { + case "SOA": content = fmt.Sprintf("%s %v %d %d %d %d ttl=%d", rc.target, rc.SoaMbox, rc.SoaRefresh, rc.SoaRetry, rc.SoaExpire, rc.SoaMinttl, rc.TTL) // SoaSerial is not used in comparison + default: + content = fmt.Sprintf("%v ttl=%d", rc.GetTargetCombined(), rc.TTL) } for _, valueMap := range extraMaps { // sort the extra values map keys to perform a deterministic @@ -424,6 +427,10 @@ type RecordKey struct { Type string } +func (rk *RecordKey) String() string { + return rk.NameFQDN + ":" + rk.Type +} + // Key converts a RecordConfig into a RecordKey. func (rc *RecordConfig) Key() RecordKey { t := rc.Type diff --git a/package-lock.json b/package-lock.json index 870d78e95..06d9d8313 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,33 @@ { + "name": "dnscontrol", + "lockfileVersion": 2, "requires": true, - "lockfileVersion": 1, + "packages": { + "": { + "dependencies": { + "prettier": "^2.8.1" + } + }, + "node_modules/prettier": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.1.tgz", + "integrity": "sha512-lqGoSJBQNJidqCHE80vqZJHWHRFoNYsSpP9AjFhlhi9ODCJA541svILes/+/1GM3VaL/abZi7cpFzOpdR9UPKg==", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + } + }, "dependencies": { "prettier": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", - "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.1.tgz", + "integrity": "sha512-lqGoSJBQNJidqCHE80vqZJHWHRFoNYsSpP9AjFhlhi9ODCJA541svILes/+/1GM3VaL/abZi7cpFzOpdR9UPKg==" } } } diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index cedd10ac9..39526baf7 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -10,7 +10,7 @@ import ( "github.com/gobwas/glob" ) -// Correlation stores a difference between two domains. +// Correlation stores a difference between two records. type Correlation struct { d *differ Existing *models.RecordConfig diff --git a/pkg/diff2/analyze.go b/pkg/diff2/analyze.go new file mode 100644 index 000000000..8c4677d57 --- /dev/null +++ b/pkg/diff2/analyze.go @@ -0,0 +1,275 @@ +package diff2 + +import ( + "fmt" + "sort" + "strings" + + "github.com/StackExchange/dnscontrol/v3/models" +) + +func analyzeByRecordSet(cc *CompareConfig) ChangeList { + var instructions ChangeList + // For each label... + for _, lc := range cc.ldata { + // for each type at that label... + for _, rt := range lc.tdata { + // ...if there are changes generate an instruction. + ets := rt.existingTargets + dts := rt.desiredTargets + msgs := genmsgs(ets, dts) + if len(msgs) == 0 { // No differences? + //fmt.Printf("DEBUG: done. Records are the same\n") + // The records at this rset are the same. No work to be done. + continue + } + if len(ets) == 0 { // Create a new label. + //fmt.Printf("DEBUG: add\n") + instructions = append(instructions, mkAdd(lc.label, rt.rType, msgs, rt.desiredRecs)) + } else if len(dts) == 0 { // Delete that label and all its records. + //fmt.Printf("DEBUG: delete\n") + instructions = append(instructions, mkDelete(lc.label, rt.rType, rt.existingRecs, msgs)) + } else { // Change the records at that label + //fmt.Printf("DEBUG: change\n") + instructions = append(instructions, mkChange(lc.label, rt.rType, msgs, rt.existingRecs, rt.desiredRecs)) + } + } + } + return instructions +} + +func analyzeByLabel(cc *CompareConfig) ChangeList { + var instructions ChangeList + //fmt.Printf("DEBUG: START: analyzeByLabel\n") + // Accumulate if there are any changes and collect the info needed to generate instructions. + for i, lc := range cc.ldata { + //fmt.Printf("DEBUG: START LABEL = %q\n", lc.label) + label := lc.label + var accMsgs []string + var accExisting models.Records + var accDesired models.Records + msgsByKey := map[models.RecordKey][]string{} + for _, rt := range lc.tdata { + //fmt.Printf("DEBUG: START RTYPE = %q\n", rt.rType) + ets := rt.existingTargets + dts := rt.desiredTargets + msgs := genmsgs(ets, dts) + msgsByKey[models.RecordKey{NameFQDN: label, Type: rt.rType}] = msgs + //fmt.Printf("DEBUG: appending msgs=%v\n", msgs) + accMsgs = append(accMsgs, msgs...) // Accumulate the messages + accExisting = append(accExisting, rt.existingRecs...) // Accumulate records existing at this label. + accDesired = append(accDesired, rt.desiredRecs...) // Accumulate records desired at this label. + } + + // We now know what changed (accMsgs), + // what records USED TO EXIST at that label (accExisting), + // and what records SHOULD EXIST at that label (accDesired). + // Based on that info, we can generate the instructions. + + if len(accMsgs) == 0 { // Nothing changed. + //fmt.Printf("DEBUG: analyzeByLabel: %02d: no change\n", i) + } else if len(accDesired) == 0 { // No new records at the label? This must be a delete. + //fmt.Printf("DEBUG: analyzeByLabel: %02d: delete\n", i) + instructions = append(instructions, mkDelete(label, "", accExisting, accMsgs)) + } else if len(accExisting) == 0 { // No old records at the label? This must be a change. + //fmt.Printf("DEBUG: analyzeByLabel: %02d: create\n", i) + fmt.Printf("DEBUG: analyzeByLabel mkAdd msgs=%d\n", len(accMsgs)) + instructions = append(instructions, mkAddByLabel(label, "", accMsgs, accDesired)) + } else { // If we get here, it must be a change. + _ = i + // fmt.Printf("DEBUG: analyzeByLabel: %02d: change %d{%v} %d{%v} msgs=%v\n", i, + // len(accExisting), accExisting, + // len(accDesired), accDesired, + // accMsgs, + // ) + fmt.Printf("DEBUG: analyzeByLabel mkchange msgs=%d\n", len(accMsgs)) + instructions = append(instructions, mkChangeLabel(label, "", accMsgs, accExisting, accDesired, msgsByKey)) + } + } + + return instructions +} + +func analyzeByRecord(cc *CompareConfig) ChangeList { + //fmt.Printf("DEBUG: analyzeByRecord: cc=%v\n", cc) + + var instructions ChangeList + // For each label, for each type at that label, see if there are any changes. + for _, lc := range cc.ldata { + //fmt.Printf("DEBUG: analyzeByRecord: next lc=%v\n", lc) + for _, rt := range lc.tdata { + ets := rt.existingTargets + dts := rt.desiredTargets + cs := diffTargets(ets, dts) + //fmt.Printf("DEBUG: analyzeByRecord: cs=%v\n", cs) + instructions = append(instructions, cs...) + } + } + return instructions +} + +// NB(tlim): there is no analyzeByZone. ByZone calls anayzeByRecords(). + +func mkAdd(l string, t string, msgs []string, recs models.Records) Change { + c := Change{Type: CREATE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.New = recs + return c +} + +// TODO(tlim): Clean these up. Some of them are exact duplicates! + +func mkAddByLabel(l string, t string, msgs []string, newRecs models.Records) Change { + fmt.Printf("DEBUG: mkAddByLabel: len(o)=%d len(m)=%d\n", len(newRecs), len(msgs)) + fmt.Printf("DEBUG: mkAddByLabel: msgs = %v\n", msgs) + c := Change{Type: CREATE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.New = newRecs + return c +} + +func mkChange(l string, t string, msgs []string, oldRecs, newRecs models.Records) Change { + c := Change{Type: CHANGE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.Old = oldRecs + c.New = newRecs + return c +} + +func mkChangeLabel(l string, t string, msgs []string, oldRecs, newRecs models.Records, msgsByKey map[models.RecordKey][]string) Change { + //fmt.Printf("DEBUG: mkChangeLabel: len(o)=%d\n", len(oldRecs)) + c := Change{Type: CHANGE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.Old = oldRecs + c.New = newRecs + c.MsgsByKey = msgsByKey + return c +} + +func mkDelete(l string, t string, oldRecs models.Records, msgs []string) Change { + c := Change{Type: DELETE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.Old = oldRecs + return c +} +func mkDeleteRec(l string, t string, msgs []string, rec *models.RecordConfig) Change { + c := Change{Type: DELETE, Msgs: msgs} + c.Key.NameFQDN = l + c.Key.Type = t + c.Old = models.Records{rec} + return c +} + +func removeCommon(existing, desired []targetConfig) ([]targetConfig, []targetConfig) { + + // NB(tlim): We could probably make this faster. Some ideas: + // * pre-allocate newexisting/newdesired and assign to indexed elements instead of appending. + // * iterate backwards (len(d) to 0) and delete items that are the same. + // On the other hand, this function typically receives lists of 1-3 elements + // and any optimization is probably fruitless. + + eKeys := map[string]*targetConfig{} + for _, v := range existing { + eKeys[v.compareable] = &v + } + dKeys := map[string]*targetConfig{} + for _, v := range desired { + dKeys[v.compareable] = &v + } + + return filterBy(existing, dKeys), filterBy(desired, eKeys) +} + +func filterBy(s []targetConfig, m map[string]*targetConfig) []targetConfig { + i := 0 // output index + for _, x := range s { + if _, ok := m[x.compareable]; !ok { + // copy and increment index + s[i] = x + i++ + } + } + // // Prevent memory leak by erasing truncated values + // // (not needed if values don't contain pointers, directly or indirectly) + // for j := i; j < len(s); j++ { + // s[j] = nil + // } + s = s[:i] + return s +} + +func diffTargets(existing, desired []targetConfig) ChangeList { + //fmt.Printf("DEBUG: diffTargets called with len(e)=%d len(d)=%d\n", len(existing), len(desired)) + + // Nothing to do? + if len(existing) == 0 && len(desired) == 0 { + //fmt.Printf("DEBUG: diffTargets: nothing to do\n") + return nil + } + + // Sort to make comparisons easier + sort.Slice(existing, func(i, j int) bool { return existing[i].compareable < existing[j].compareable }) + sort.Slice(desired, func(i, j int) bool { return desired[i].compareable < desired[j].compareable }) + + var instructions ChangeList + + // remove the exact matches. + existing, desired = removeCommon(existing, desired) + + // the common chunk are changes + mi := min(len(existing), len(desired)) + //fmt.Printf("DEBUG: min=%d\n", mi) + for i := 0; i < mi; i++ { + //fmt.Println(i, "CHANGE") + er := existing[i].rec + dr := desired[i].rec + m := fmt.Sprintf("CHANGE %s %s (%s) -> (%s)", dr.NameFQDN, dr.Type, er.GetTargetCombined(), dr.GetTargetCombined()) + instructions = append(instructions, mkChange(dr.NameFQDN, dr.Type, []string{m}, + //models.Records{existing[i].rec}, + //models.Records{desired[i].rec}, + models.Records{er}, + models.Records{dr}, + )) + } + + // any left-over existing are deletes + for i := mi; i < len(existing); i++ { + //fmt.Println(i, "DEL") + er := existing[i].rec + m := fmt.Sprintf("DELETE %s %s %s", er.NameFQDN, er.Type, er.GetTargetCombined()) + instructions = append(instructions, mkDeleteRec(er.NameFQDN, er.Type, []string{m}, er)) + } + + // any left-over desired are creates + for i := mi; i < len(desired); i++ { + //fmt.Println(i, "CREATE") + dr := desired[i].rec + m := fmt.Sprintf("CREATE %s %s %s", dr.NameFQDN, dr.Type, dr.GetTargetCombined()) + instructions = append(instructions, mkAdd(dr.NameFQDN, dr.Type, []string{m}, models.Records{dr})) + } + + return instructions +} + +func genmsgs(existing, desired []targetConfig) []string { + cl := diffTargets(existing, desired) + return justMsgs(cl) +} + +func justMsgs(cl ChangeList) []string { + var msgs []string + for _, c := range cl { + msgs = append(msgs, c.Msgs...) + } + return msgs +} + +func justMsgString(cl ChangeList) string { + msgs := justMsgs(cl) + return strings.Join(msgs, "\n") +} diff --git a/pkg/diff2/analyze_test.go b/pkg/diff2/analyze_test.go new file mode 100644 index 000000000..7a959a808 --- /dev/null +++ b/pkg/diff2/analyze_test.go @@ -0,0 +1,635 @@ +package diff2 + +import ( + "reflect" + "strings" + "testing" + + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/kylelemons/godebug/diff" +) + +var testDataAA1234 = makeRec("laba", "A", "1.2.3.4") // [0] +var testDataAA5678 = makeRec("laba", "A", "5.6.7.8") // [0] +var testDataAMX10a = makeRec("laba", "MX", "10 laba") // [1] +var testDataCCa = makeRec("labc", "CNAME", "laba") // [2] +var testDataEA15 = makeRec("labe", "A", "10.10.10.15") // [3] +var e4 = makeRec("labe", "A", "10.10.10.16") // [4] +var e5 = makeRec("labe", "A", "10.10.10.17") // [5] +var e6 = makeRec("labe", "A", "10.10.10.18") // [6] +var e7 = makeRec("labg", "NS", "10.10.10.15") // [7] +var e8 = makeRec("labg", "NS", "10.10.10.16") // [8] +var e9 = makeRec("labg", "NS", "10.10.10.17") // [9] +var e10 = makeRec("labg", "NS", "10.10.10.18") // [10] +var e11mx = makeRec("labh", "MX", "22 ttt") // [11] +var e11 = makeRec("labh", "CNAME", "labd") // [11] +var testDataApexMX1aaa = makeRec("", "MX", "1 aaa") + +var testDataAA1234clone = makeRec("laba", "A", "1.2.3.4") // [0'] +var testDataAA12345 = makeRec("laba", "A", "1.2.3.5") // [1'] +var testDataAMX20b = makeRec("laba", "MX", "20 labb") // [2'] +var d3 = makeRec("labe", "A", "10.10.10.95") // [3'] +var d4 = makeRec("labe", "A", "10.10.10.96") // [4'] +var d5 = makeRec("labe", "A", "10.10.10.97") // [5'] +var d6 = makeRec("labe", "A", "10.10.10.98") // [6'] +var d7 = makeRec("labf", "TXT", "foo") // [7'] +var d8 = makeRec("labg", "NS", "10.10.10.10") // [8'] +var d9 = makeRec("labg", "NS", "10.10.10.15") // [9'] +var d10 = makeRec("labg", "NS", "10.10.10.16") // [10'] +var d11 = makeRec("labg", "NS", "10.10.10.97") // [11'] +var d12 = makeRec("labh", "A", "1.2.3.4") // [12'] +var testDataApexMX22bbb = makeRec("", "MX", "22 bbb") + +var d0tc = targetConfig{compareable: "1.2.3.4 ttl=0", rec: testDataAA1234clone} + +func makeChange(v Verb, l, t string, old, new models.Records, msgs []string) Change { + c := Change{ + Type: v, + Old: old, + New: new, + Msgs: msgs, + } + c.Key.NameFQDN = l + c.Key.Type = t + return c +} + +func compareMsgs(t *testing.T, fnname, testname, testpart string, gotcc ChangeList, wantstring string) { + t.Helper() + gs := strings.TrimSpace(justMsgString(gotcc)) + ws := strings.TrimSpace(wantstring) + d := diff.Diff(gs, ws) + if d != "" { + t.Errorf("%s()/%s (wantMsgs:%s):\n===got===\n%s\n===want===\n%s\n===diff===\n%s\n===", fnname, testname, testpart, gs, ws, d) + } +} + +func compareCL(t *testing.T, fnname, testname, testpart string, gotcl ChangeList, wantstring string) { + t.Helper() + gs := strings.TrimSpace(gotcl.String()) + ws := strings.TrimSpace(wantstring) + d := diff.Diff(gs, ws) + if d != "" { + t.Errorf("%s()/%s (wantChange%s):\n===got===\n%s\n===want===\n%s\n===diff===\n%s\n===", fnname, testname, testpart, gs, ws, d) + } + +} + +func Test_analyzeByRecordSet(t *testing.T) { + type args struct { + origin string + existing, desired models.Records + compFn ComparableFunc + } + + origin := "f.com" + existing := models.Records{testDataAA1234, testDataAMX10a, testDataCCa, testDataEA15, e4, e5, e6, e7, e8, e9, e10, e11} + desired := models.Records{testDataAA1234clone, testDataAA12345, testDataAMX20b, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12} + + tests := []struct { + name string + args args + wantMsgs string + wantChangeRSet string + wantChangeLabel string + wantChangeRec string + wantChangeZone string + }{ + + { + name: "oneequal", + args: args{ + origin: origin, + existing: models.Records{testDataAA1234}, + desired: models.Records{testDataAA1234clone}, + }, + wantMsgs: "", // Empty + wantChangeRSet: "ChangeList: len=0", + wantChangeLabel: "ChangeList: len=0", + wantChangeRec: "ChangeList: len=0", + }, + + { + name: "onediff", + args: args{ + origin: origin, + existing: models.Records{testDataAA1234, testDataAMX10a}, + desired: models.Records{testDataAA1234clone, testDataAMX20b}, + }, + wantMsgs: "CHANGE laba.f.com MX (10 laba) -> (20 labb)", + wantChangeRSet: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +`, + wantChangeLabel: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={laba.f.com } + old=[1.2.3.4 10 laba] + new=[1.2.3.4 20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +`, + wantChangeRec: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +`, + }, + + { + name: "apex", + args: args{ + origin: origin, + existing: models.Records{testDataAA1234, testDataApexMX1aaa}, + desired: models.Records{testDataAA1234clone, testDataApexMX22bbb}, + }, + wantMsgs: "CHANGE f.com MX (1 aaa) -> (22 bbb)", + wantChangeRSet: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={f.com MX} + old=[1 aaa] + new=[22 bbb] + msg=["CHANGE f.com MX (1 aaa) -> (22 bbb)"] +`, + wantChangeLabel: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={f.com } + old=[1 aaa] + new=[22 bbb] + msg=["CHANGE f.com MX (1 aaa) -> (22 bbb)"] +`, + wantChangeRec: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={f.com MX} + old=[1 aaa] + new=[22 bbb] + msg=["CHANGE f.com MX (1 aaa) -> (22 bbb)"] +`, + }, + + { + name: "firsta", + args: args{ + origin: origin, + existing: models.Records{testDataAA1234, testDataAMX10a}, + desired: models.Records{testDataAA1234clone, testDataAA12345, testDataAMX20b}, + }, + wantMsgs: ` +CREATE laba.f.com A 1.2.3.5 +CHANGE laba.f.com MX (10 laba) -> (20 labb) + `, + wantChangeRSet: ` +ChangeList: len=2 +00: Change: verb=CHANGE + key={laba.f.com A} + old=[1.2.3.4] + new=[1.2.3.4 1.2.3.5] + msg=["CREATE laba.f.com A 1.2.3.5"] +01: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] + `, + wantChangeLabel: ` +ChangeList: len=1 +00: Change: verb=CHANGE + key={laba.f.com } + old=[1.2.3.4 10 laba] + new=[1.2.3.4 1.2.3.5 20 labb] + msg=["CREATE laba.f.com A 1.2.3.5" "CHANGE laba.f.com MX (10 laba) -> (20 labb)"] + `, + wantChangeRec: ` +ChangeList: len=2 +00: Change: verb=CREATE + key={laba.f.com A} + new=[1.2.3.5] + msg=["CREATE laba.f.com A 1.2.3.5"] +01: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +`, + }, + + { + name: "big", + args: args{ + origin: origin, + existing: existing, + desired: desired, + }, + wantMsgs: ` +CREATE laba.f.com A 1.2.3.5 +CHANGE laba.f.com MX (10 laba) -> (20 labb) +DELETE labc.f.com CNAME laba +CHANGE labe.f.com A (10.10.10.15) -> (10.10.10.95) +CHANGE labe.f.com A (10.10.10.16) -> (10.10.10.96) +CHANGE labe.f.com A (10.10.10.17) -> (10.10.10.97) +CHANGE labe.f.com A (10.10.10.18) -> (10.10.10.98) +CREATE labf.f.com TXT "foo" +CHANGE labg.f.com NS (10.10.10.17) -> (10.10.10.10) +CHANGE labg.f.com NS (10.10.10.18) -> (10.10.10.97) +DELETE labh.f.com CNAME labd +CREATE labh.f.com A 1.2.3.4 + `, + wantChangeRSet: ` +ChangeList: len=8 +00: Change: verb=CHANGE + key={laba.f.com A} + old=[1.2.3.4] + new=[1.2.3.4 1.2.3.5] + msg=["CREATE laba.f.com A 1.2.3.5"] +01: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +02: Change: verb=DELETE + key={labc.f.com CNAME} + old=[laba] + msg=["DELETE labc.f.com CNAME laba"] +03: Change: verb=CHANGE + key={labe.f.com A} + old=[10.10.10.15 10.10.10.16 10.10.10.17 10.10.10.18] + new=[10.10.10.95 10.10.10.96 10.10.10.97 10.10.10.98] + msg=["CHANGE labe.f.com A (10.10.10.15) -> (10.10.10.95)" "CHANGE labe.f.com A (10.10.10.16) -> (10.10.10.96)" "CHANGE labe.f.com A (10.10.10.17) -> (10.10.10.97)" "CHANGE labe.f.com A (10.10.10.18) -> (10.10.10.98)"] +04: Change: verb=CREATE + key={labf.f.com TXT} + new=["foo"] + msg=["CREATE labf.f.com TXT \"foo\""] +05: Change: verb=CHANGE + key={labg.f.com NS} + old=[10.10.10.15 10.10.10.16 10.10.10.17 10.10.10.18] + new=[10.10.10.10 10.10.10.15 10.10.10.16 10.10.10.97] + msg=["CHANGE labg.f.com NS (10.10.10.17) -> (10.10.10.10)" "CHANGE labg.f.com NS (10.10.10.18) -> (10.10.10.97)"] +06: Change: verb=DELETE + key={labh.f.com CNAME} + old=[labd] + msg=["DELETE labh.f.com CNAME labd"] +07: Change: verb=CREATE + key={labh.f.com A} + new=[1.2.3.4] + msg=["CREATE labh.f.com A 1.2.3.4"] + `, + wantChangeLabel: ` +ChangeList: len=6 +00: Change: verb=CHANGE + key={laba.f.com } + old=[1.2.3.4 10 laba] + new=[1.2.3.4 1.2.3.5 20 labb] + msg=["CREATE laba.f.com A 1.2.3.5" "CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +01: Change: verb=DELETE + key={labc.f.com } + old=[laba] + msg=["DELETE labc.f.com CNAME laba"] +02: Change: verb=CHANGE + key={labe.f.com } + old=[10.10.10.15 10.10.10.16 10.10.10.17 10.10.10.18] + new=[10.10.10.95 10.10.10.96 10.10.10.97 10.10.10.98] + msg=["CHANGE labe.f.com A (10.10.10.15) -> (10.10.10.95)" "CHANGE labe.f.com A (10.10.10.16) -> (10.10.10.96)" "CHANGE labe.f.com A (10.10.10.17) -> (10.10.10.97)" "CHANGE labe.f.com A (10.10.10.18) -> (10.10.10.98)"] +03: Change: verb=CREATE + key={labf.f.com } + new=["foo"] + msg=["CREATE labf.f.com TXT \"foo\""] +04: Change: verb=CHANGE + key={labg.f.com } + old=[10.10.10.15 10.10.10.16 10.10.10.17 10.10.10.18] + new=[10.10.10.10 10.10.10.15 10.10.10.16 10.10.10.97] + msg=["CHANGE labg.f.com NS (10.10.10.17) -> (10.10.10.10)" "CHANGE labg.f.com NS (10.10.10.18) -> (10.10.10.97)"] +05: Change: verb=CHANGE + key={labh.f.com } + old=[labd] + new=[1.2.3.4] + msg=["DELETE labh.f.com CNAME labd" "CREATE labh.f.com A 1.2.3.4"] + `, + wantChangeRec: ` +ChangeList: len=12 +00: Change: verb=CREATE + key={laba.f.com A} + new=[1.2.3.5] + msg=["CREATE laba.f.com A 1.2.3.5"] +01: Change: verb=CHANGE + key={laba.f.com MX} + old=[10 laba] + new=[20 labb] + msg=["CHANGE laba.f.com MX (10 laba) -> (20 labb)"] +02: Change: verb=DELETE + key={labc.f.com CNAME} + old=[laba] + msg=["DELETE labc.f.com CNAME laba"] +03: Change: verb=CHANGE + key={labe.f.com A} + old=[10.10.10.15] + new=[10.10.10.95] + msg=["CHANGE labe.f.com A (10.10.10.15) -> (10.10.10.95)"] +04: Change: verb=CHANGE + key={labe.f.com A} + old=[10.10.10.16] + new=[10.10.10.96] + msg=["CHANGE labe.f.com A (10.10.10.16) -> (10.10.10.96)"] +05: Change: verb=CHANGE + key={labe.f.com A} + old=[10.10.10.17] + new=[10.10.10.97] + msg=["CHANGE labe.f.com A (10.10.10.17) -> (10.10.10.97)"] +06: Change: verb=CHANGE + key={labe.f.com A} + old=[10.10.10.18] + new=[10.10.10.98] + msg=["CHANGE labe.f.com A (10.10.10.18) -> (10.10.10.98)"] +07: Change: verb=CREATE + key={labf.f.com TXT} + new=["foo"] + msg=["CREATE labf.f.com TXT \"foo\""] +08: Change: verb=CHANGE + key={labg.f.com NS} + old=[10.10.10.17] + new=[10.10.10.10] + msg=["CHANGE labg.f.com NS (10.10.10.17) -> (10.10.10.10)"] +09: Change: verb=CHANGE + key={labg.f.com NS} + old=[10.10.10.18] + new=[10.10.10.97] + msg=["CHANGE labg.f.com NS (10.10.10.18) -> (10.10.10.97)"] +10: Change: verb=DELETE + key={labh.f.com CNAME} + old=[labd] + msg=["DELETE labh.f.com CNAME labd"] +11: Change: verb=CREATE + key={labh.f.com A} + new=[1.2.3.4] + msg=["CREATE labh.f.com A 1.2.3.4"] +`, + }, + } + for _, tt := range tests { + + // Each "analyze*()" should return the same msgs, but a different ChangeList. + // Sadly the analyze*() functions are destructive to the CompareConfig struct. + // Therefore we have to run NewCompareConfig() each time. + + t.Run(tt.name, func(t *testing.T) { + cl := analyzeByRecordSet(NewCompareConfig(tt.args.origin, tt.args.existing, tt.args.desired, tt.args.compFn)) + compareMsgs(t, "analyzeByRecordSet", tt.name, "RSet", cl, tt.wantMsgs) + compareCL(t, "analyzeByRecordSet", tt.name, "RSet", cl, tt.wantChangeRSet) + }) + + t.Run(tt.name, func(t *testing.T) { + cl := analyzeByLabel(NewCompareConfig(tt.args.origin, tt.args.existing, tt.args.desired, tt.args.compFn)) + compareMsgs(t, "analyzeByLabel", tt.name, "Label", cl, tt.wantMsgs) + compareCL(t, "analyzeByLabel", tt.name, "Label", cl, tt.wantChangeLabel) + }) + + t.Run(tt.name, func(t *testing.T) { + cl := analyzeByRecord(NewCompareConfig(tt.args.origin, tt.args.existing, tt.args.desired, tt.args.compFn)) + compareMsgs(t, "analyzeByRecord", tt.name, "Rec", cl, tt.wantMsgs) + compareCL(t, "analyzeByRecord", tt.name, "Rec", cl, tt.wantChangeRec) + }) + + // NB(tlim): There is no analyzeByZone(). ByZone() uses analyzeByRecord(). + + } +} +func Test_diffTargets(t *testing.T) { + type args struct { + existing []targetConfig + desired []targetConfig + } + tests := []struct { + name string + args args + want ChangeList + }{ + + { + name: "single", + args: args{ + existing: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + desired: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + }, + //want: , + }, + + { + name: "add1", + args: args{ + existing: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + desired: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + }, + want: ChangeList{ + Change{Type: CREATE, + Key: models.RecordKey{NameFQDN: "laba.f.com", Type: "MX"}, + New: models.Records{testDataAMX10a}, + Msgs: []string{"CREATE laba.f.com MX 10 laba"}, + }, + }, + }, + + { + name: "del1", + args: args{ + existing: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + desired: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + }, + want: ChangeList{ + Change{Type: DELETE, + Key: models.RecordKey{NameFQDN: "laba.f.com", Type: "MX"}, + Old: models.Records{testDataAMX10a}, + Msgs: []string{"DELETE laba.f.com MX 10 laba"}, + }, + }, + }, + + { + name: "change2nd", + args: args{ + existing: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + desired: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "20 laba", rec: testDataAMX20b}, + }, + }, + want: ChangeList{ + Change{Type: CHANGE, + Key: models.RecordKey{NameFQDN: "laba.f.com", Type: "MX"}, + Old: models.Records{testDataAMX10a}, + New: models.Records{testDataAMX20b}, + Msgs: []string{"CHANGE laba.f.com MX (10 laba) -> (20 labb)"}, + }, + }, + }, + + { + name: "del2nd", + args: args{ + existing: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "5.6.7.8", rec: testDataAA5678}, + }, + desired: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + }, + want: ChangeList{ + Change{Type: CHANGE, + Key: models.RecordKey{NameFQDN: "laba.f.com", Type: "A"}, + Old: models.Records{testDataAA1234, testDataAA5678}, + New: models.Records{testDataAA1234}, + Msgs: []string{"DELETE laba.f.com A 5.6.7.8"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + //fmt.Printf("DEBUG: Test %02d\n", i) + got := diffTargets(tt.args.existing, tt.args.desired) + d := diff.Diff(strings.TrimSpace(justMsgString(got)), strings.TrimSpace(justMsgString(tt.want))) + if d != "" { + //fmt.Printf("DEBUG: %d %d\n", len(got), len(tt.want)) + t.Errorf("diffTargets()\n diff=%s", d) + } + }) + } +} + +func Test_removeCommon(t *testing.T) { + type args struct { + existing []targetConfig + desired []targetConfig + } + tests := []struct { + name string + args args + want []targetConfig + want1 []targetConfig + }{ + { + name: "same", + args: args{ + existing: []targetConfig{d0tc}, + desired: []targetConfig{d0tc}, + }, + want: []targetConfig{}, + want1: []targetConfig{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := removeCommon(tt.args.existing, tt.args.desired) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("removeCommon() got = %v, want %v", got, tt.want) + } + if (!(got1 == nil && tt.want1 == nil)) && !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("removeCommon() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func comparables(s []targetConfig) []string { + var r []string + for _, j := range s { + r = append(r, j.compareable) + } + return r +} + +func Test_filterBy(t *testing.T) { + type args struct { + s []targetConfig + m map[string]*targetConfig + } + tests := []struct { + name string + args args + want []targetConfig + }{ + + { + name: "removeall", + args: args{ + s: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + m: map[string]*targetConfig{ + "1.2.3.4": {compareable: "1.2.3.4", rec: testDataAA1234}, + "10 laba": {compareable: "10 laba", rec: testDataAMX10a}, + }, + }, + want: []targetConfig{}, + }, + + { + name: "keepall", + args: args{ + s: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + m: map[string]*targetConfig{ + "nothing": {compareable: "1.2.3.4", rec: testDataAA1234}, + "matches": {compareable: "10 laba", rec: testDataAMX10a}, + }, + }, + want: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + }, + + { + name: "keepsome", + args: args{ + s: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + {compareable: "10 laba", rec: testDataAMX10a}, + }, + m: map[string]*targetConfig{ + "nothing": {compareable: "1.2.3.4", rec: testDataAA1234}, + "10 laba": {compareable: "10 laba", rec: testDataAMX10a}, + }, + }, + want: []targetConfig{ + {compareable: "1.2.3.4", rec: testDataAA1234}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filterBy(tt.args.s, tt.args.m); !reflect.DeepEqual(comparables(got), comparables(tt.want)) { + t.Errorf("filterBy() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/diff2/compareconfig.go b/pkg/diff2/compareconfig.go new file mode 100644 index 000000000..368f352f9 --- /dev/null +++ b/pkg/diff2/compareconfig.go @@ -0,0 +1,214 @@ +package diff2 + +import ( + "bytes" + "fmt" + "sort" + + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/StackExchange/dnscontrol/v3/pkg/prettyzone" +) + +type ComparableFunc func(*models.RecordConfig) string + +type CompareConfig struct { + existing, desired models.Records + ldata []*labelConfig + // + origin string // Domain zone + compareableFunc ComparableFunc + // + labelMap map[string]bool + keyMap map[models.RecordKey]bool +} + +type labelConfig struct { + label string + tdata []*rTypeConfig +} + +type rTypeConfig struct { + rType string + existingTargets []targetConfig + desiredTargets []targetConfig + existingRecs []*models.RecordConfig + desiredRecs []*models.RecordConfig +} + +type targetConfig struct { + compareable string + rec *models.RecordConfig +} + +func NewCompareConfig(origin string, existing, desired models.Records, compFn ComparableFunc) *CompareConfig { + cc := &CompareConfig{ + existing: existing, + desired: desired, + // + origin: origin, + compareableFunc: compFn, + // + labelMap: map[string]bool{}, + keyMap: map[models.RecordKey]bool{}, + } + cc.addRecords(existing, true) + cc.addRecords(desired, false) + cc.VerifyCNAMEAssertions() + sort.Slice(cc.ldata, func(i, j int) bool { + return prettyzone.LabelLess(cc.ldata[i].label, cc.ldata[j].label) + }) + return cc +} + +func (cc *CompareConfig) VerifyCNAMEAssertions() { + + // NB(tlim): This can be deleted. This should be probably not possible. + // However, let's keep it around for a few iterations to be paranoid. + + // According to the RFCs if a label has a CNAME, it can not have any other + // records at that label... even other CNAMEs. Therefore, we need to be + // careful with changes at a label that involve a CNAME. + // Example 1: + // OLD: a.example.com CNAME b + // NEW: a.example.com A 1.2.3.4 + // We must delete the CNAME record THEN create the A record. If we + // blindly create the the A first, most APIs will reply with an error + // because there is already a CNAME at that label. + // Example 2: + // OLD: a.example.com A 1.2.3.4 + // NEW: a.example.com CNAME b + // We must delete the A record THEN create the CNAME. If we blindly + // create the CNAME first, most APIs will reply with an error because + // there is already an A record at that label. + // + // To assure that DNS providers don't have to think about this, we order + // the tdata items so that we generate the instructions in the best order. + // In other words: + // If there is a CNAME in existing, it should be in front. + // If there is a CNAME in desired, it should be at the end. + + // That said, since we addRecords existing first, and desired last, the data + // should already be in the right order. + + for _, ld := range cc.ldata { + for j, td := range ld.tdata { + if td.rType == "CNAME" { + if len(td.existingTargets) != 0 { + //fmt.Printf("DEBUG: cname in existing: index=%d\n", j) + if j != 0 { + panic("should not happen: (CNAME not in first position)") + } + } + if len(td.desiredTargets) != 0 { + //fmt.Printf("DEBUG: cname in desired: index=%d\n", j) + if j != highest(ld.tdata) { + panic("should not happen: (CNAME not in last position)") + } + } + } + } + } + +} + +func (cc *CompareConfig) String() string { + var buf bytes.Buffer + b := &buf + + fmt.Fprintf(b, "ldata:\n") + for i, ld := range cc.ldata { + fmt.Fprintf(b, " ldata[%02d]: %s\n", i, ld.label) + for j, t := range ld.tdata { + fmt.Fprintf(b, " tdata[%d]: %q e(%d, %d) d(%d, %d)\n", j, t.rType, + len(t.existingTargets), + len(t.existingRecs), + len(t.desiredTargets), + len(t.desiredRecs), + ) + } + } + fmt.Fprintf(b, "labelMap: len=%d %v\n", len(cc.labelMap), cc.labelMap) + fmt.Fprintf(b, "keyMap: len=%d %v\n", len(cc.keyMap), cc.keyMap) + fmt.Fprintf(b, "existing: %q\n", cc.existing) + fmt.Fprintf(b, "desired: %q\n", cc.desired) + fmt.Fprintf(b, "origin: %v\n", cc.origin) + fmt.Fprintf(b, "compFn: %v\n", cc.compareableFunc) + + return b.String() +} + +func comparable(rc *models.RecordConfig, f func(*models.RecordConfig) string) string { + if f == nil { + return rc.ToDiffable() + } + return rc.ToDiffable() + " " + f(rc) +} + +func (cc *CompareConfig) addRecords(recs models.Records, storeInExisting bool) { + + // Sort, because sorted data is easier to work with. + // NB(tlim): The actual sort order doesn't matter as long as all the records + // of the same label+rtype are adjacent. We use PrettySort because it works, + // has been extensively tested, and assures that the ChangeList will be a + // pretty order. + //for _, rec := range recs { + z := prettyzone.PrettySort(recs, cc.origin, 0, nil) + for _, rec := range z.Records { + + label := rec.NameFQDN + rtype := rec.Type + comp := comparable(rec, cc.compareableFunc) + //fmt.Printf("DEBUG addRecords rec=%v:%v es=%v comp=%v\n", label, rtype, storeInExisting, comp) + + //fmt.Printf("BEFORE L: %v\n", len(cc.ldata)) + // Are we seeing this label for the first time? + var labelIdx int + if _, ok := cc.labelMap[label]; !ok { + //fmt.Printf("DEBUG: I haven't see label=%v before. Adding.\n", label) + cc.labelMap[label] = true + cc.ldata = append(cc.ldata, &labelConfig{label: label}) + labelIdx = highest(cc.ldata) + } else { + // find label in cc.ldata: + for k, v := range cc.ldata { + if v.label == label { + labelIdx = k + break + } + } + } + + // Are we seeing this label+rtype for the first time? + key := rec.Key() + if _, ok := cc.keyMap[key]; !ok { + //fmt.Printf("DEBUG: I haven't see key=%v before. Adding.\n", key) + cc.keyMap[key] = true + x := cc.ldata[labelIdx] + //fmt.Printf("DEBUG: appending rtype=%v\n", rtype) + x.tdata = append(x.tdata, &rTypeConfig{rType: rtype}) + } + var rtIdx int + // find rtype in tdata: + for k, v := range cc.ldata[labelIdx].tdata { + if v.rType == rtype { + rtIdx = k + break + } + } + //fmt.Printf("DEBUG: found rtype=%v at index %d\n", rtype, rtIdx) + + // Now it is safe to add/modify the records. + + //fmt.Printf("BEFORE E/D: %v/%v\n", len(td.existingRecs), len(td.desiredRecs)) + if storeInExisting { + cc.ldata[labelIdx].tdata[rtIdx].existingRecs = append(cc.ldata[labelIdx].tdata[rtIdx].existingRecs, rec) + cc.ldata[labelIdx].tdata[rtIdx].existingTargets = append(cc.ldata[labelIdx].tdata[rtIdx].existingTargets, targetConfig{compareable: comp, rec: rec}) + } else { + cc.ldata[labelIdx].tdata[rtIdx].desiredRecs = append(cc.ldata[labelIdx].tdata[rtIdx].desiredRecs, rec) + cc.ldata[labelIdx].tdata[rtIdx].desiredTargets = append(cc.ldata[labelIdx].tdata[rtIdx].desiredTargets, targetConfig{compareable: comp, rec: rec}) + } + //fmt.Printf("AFTER L: %v\n", len(cc.ldata)) + //fmt.Printf("AFTER E/D: %v/%v\n", len(td.existingRecs), len(td.desiredRecs)) + //fmt.Printf("\n") + } +} diff --git a/pkg/diff2/compareconfig_test.go b/pkg/diff2/compareconfig_test.go new file mode 100644 index 000000000..6c807f751 --- /dev/null +++ b/pkg/diff2/compareconfig_test.go @@ -0,0 +1,214 @@ +package diff2 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/kylelemons/godebug/diff" +) + +func prettyPrint(i interface{}) string { + s, _ := json.MarshalIndent(i, "", "\t") + return string(s) +} + +func TestNewCompareConfig(t *testing.T) { + type args struct { + origin string + existing models.Records + desired models.Records + compFn ComparableFunc + } + tests := []struct { + name string + args args + want string + }{ + + { + name: "one", + args: args{ + origin: "f.com", + existing: models.Records{testDataAA1234}, + desired: models.Records{testDataAA1234clone}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: laba.f.com + tdata[0]: "A" e(1, 1) d(1, 1) +labelMap: len=1 map[laba.f.com:true] +keyMap: len=1 map[{laba.f.com A}:true] +existing: ["1.2.3.4"] +desired: ["1.2.3.4"] +origin: f.com +compFn: + `, + }, + + { + name: "cnameAdd", + args: args{ + origin: "f.com", + existing: models.Records{e11mx, d12}, + desired: models.Records{e11}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: labh.f.com + tdata[0]: "A" e(1, 1) d(0, 0) + tdata[1]: "MX" e(1, 1) d(0, 0) + tdata[2]: "CNAME" e(0, 0) d(1, 1) +labelMap: len=1 map[labh.f.com:true] +keyMap: len=3 map[{labh.f.com A}:true {labh.f.com CNAME}:true {labh.f.com MX}:true] +existing: ["22 ttt" "1.2.3.4"] +desired: ["labd"] +origin: f.com +compFn: + `, + }, + + { + name: "cnameDel", + args: args{ + origin: "f.com", + existing: models.Records{e11}, + desired: models.Records{d12, e11mx}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: labh.f.com + tdata[0]: "CNAME" e(1, 1) d(0, 0) + tdata[1]: "A" e(0, 0) d(1, 1) + tdata[2]: "MX" e(0, 0) d(1, 1) +labelMap: len=1 map[labh.f.com:true] +keyMap: len=3 map[{labh.f.com A}:true {labh.f.com CNAME}:true {labh.f.com MX}:true] +existing: ["labd"] +desired: ["1.2.3.4" "22 ttt"] +origin: f.com +compFn: + `, + }, + + { + name: "two", + args: args{ + origin: "f.com", + existing: models.Records{testDataAA1234, testDataCCa}, + desired: models.Records{testDataAA1234clone}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: laba.f.com + tdata[0]: "A" e(1, 1) d(1, 1) + ldata[01]: labc.f.com + tdata[0]: "CNAME" e(1, 1) d(0, 0) +labelMap: len=2 map[laba.f.com:true labc.f.com:true] +keyMap: len=2 map[{laba.f.com A}:true {labc.f.com CNAME}:true] +existing: ["1.2.3.4" "laba"] +desired: ["1.2.3.4"] +origin: f.com +compFn: +`, + }, + + { + name: "apex", + args: args{ + origin: "f.com", + existing: models.Records{testDataAA1234, testDataApexMX1aaa}, + desired: models.Records{testDataAA1234clone, testDataApexMX22bbb}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: f.com + tdata[0]: "MX" e(1, 1) d(1, 1) + ldata[01]: laba.f.com + tdata[0]: "A" e(1, 1) d(1, 1) +labelMap: len=2 map[f.com:true laba.f.com:true] +keyMap: len=2 map[{f.com MX}:true {laba.f.com A}:true] +existing: ["1.2.3.4" "1 aaa"] +desired: ["1.2.3.4" "22 bbb"] +origin: f.com +compFn: +`, + }, + + { + name: "many", + args: args{ + origin: "f.com", + existing: models.Records{testDataAA1234, testDataAMX10a, testDataCCa, testDataEA15}, + desired: models.Records{testDataAA1234clone, testDataAA12345, testDataAMX20b, d3, d4}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: laba.f.com + tdata[0]: "A" e(1, 1) d(2, 2) + tdata[1]: "MX" e(1, 1) d(1, 1) + ldata[01]: labc.f.com + tdata[0]: "CNAME" e(1, 1) d(0, 0) + ldata[02]: labe.f.com + tdata[0]: "A" e(1, 1) d(2, 2) +labelMap: len=3 map[laba.f.com:true labc.f.com:true labe.f.com:true] +keyMap: len=4 map[{laba.f.com A}:true {laba.f.com MX}:true {labc.f.com CNAME}:true {labe.f.com A}:true] +existing: ["1.2.3.4" "10 laba" "laba" "10.10.10.15"] +desired: ["1.2.3.4" "1.2.3.5" "20 labb" "10.10.10.95" "10.10.10.96"] +origin: f.com +compFn: +`, + }, + + { + name: "all", + args: args{ + origin: "f.com", + existing: models.Records{testDataAA1234, testDataAMX10a, testDataCCa, testDataEA15, e4, e5, e6, e7, e8, e9, e10, e11}, + desired: models.Records{testDataAA1234clone, testDataAA12345, testDataAMX20b, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12}, + compFn: nil, + }, + want: ` +ldata: + ldata[00]: laba.f.com + tdata[0]: "A" e(1, 1) d(2, 2) + tdata[1]: "MX" e(1, 1) d(1, 1) + ldata[01]: labc.f.com + tdata[0]: "CNAME" e(1, 1) d(0, 0) + ldata[02]: labe.f.com + tdata[0]: "A" e(4, 4) d(4, 4) + ldata[03]: labf.f.com + tdata[0]: "TXT" e(0, 0) d(1, 1) + ldata[04]: labg.f.com + tdata[0]: "NS" e(4, 4) d(4, 4) + ldata[05]: labh.f.com + tdata[0]: "CNAME" e(1, 1) d(0, 0) + tdata[1]: "A" e(0, 0) d(1, 1) +labelMap: len=6 map[laba.f.com:true labc.f.com:true labe.f.com:true labf.f.com:true labg.f.com:true labh.f.com:true] +keyMap: len=8 map[{laba.f.com A}:true {laba.f.com MX}:true {labc.f.com CNAME}:true {labe.f.com A}:true {labf.f.com TXT}:true {labg.f.com NS}:true {labh.f.com A}:true {labh.f.com CNAME}:true] +existing: ["1.2.3.4" "10 laba" "laba" "10.10.10.15" "10.10.10.16" "10.10.10.17" "10.10.10.18" "10.10.10.15" "10.10.10.16" "10.10.10.17" "10.10.10.18" "labd"] +desired: ["1.2.3.4" "1.2.3.5" "20 labb" "10.10.10.95" "10.10.10.96" "10.10.10.97" "10.10.10.98" "\"foo\"" "10.10.10.10" "10.10.10.15" "10.10.10.16" "10.10.10.97" "1.2.3.4"] +origin: f.com +compFn: +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cc := NewCompareConfig(tt.args.origin, tt.args.existing, tt.args.desired, tt.args.compFn) + got := strings.TrimSpace(cc.String()) + tt.want = strings.TrimSpace(tt.want) + if got != tt.want { + d := diff.Diff(got, tt.want) + t.Errorf("NewCompareConfig() = \n%s\n", d) + } + }) + } +} diff --git a/pkg/diff2/diff2.go b/pkg/diff2/diff2.go new file mode 100644 index 000000000..a3c9baa35 --- /dev/null +++ b/pkg/diff2/diff2.go @@ -0,0 +1,185 @@ +package diff2 + +//go:generate stringer -type=Verb + +// This module provides functions that "diff" the existing records +// against the desired records. + +import ( + "bytes" + "fmt" + + "github.com/StackExchange/dnscontrol/v3/models" +) + +type Verb int + +const ( + _ Verb = iota // Skip the first value of 0 + CREATE // Create a record/recordset/label where none existed before. + CHANGE // Change existing record/recordset/label + DELETE // Delete existing record/recordset/label +) + +type ChangeList []Change + +type Change struct { + Type Verb // Add, Change, Delete + + Key models.RecordKey // .Key.Type is "" unless using ByRecordSet + Old models.Records + New models.Records // any changed or added records at Key. + Msgs []string // Human-friendly explanation of what changed + MsgsByKey map[models.RecordKey][]string // Messages for a given key +} + +// ByRecordSet takes two lists of records (existing and desired) and +// returns instructions for turning existing into desired. +// +// Use this with DNS providers whose API updates one recordset at a +// time. A recordset is all the records of a particular type at a +// label. For example, if www.example.com has 3 A records and a TXT +// record, if A records are added, changed, or removed, the API takes +// www.example.com, A, and a list of all the desired IP addresses. +// +// Examples include: +func ByRecordSet(existing models.Records, dc *models.DomainConfig, compFunc ComparableFunc) (ChangeList, error) { + // dc stores the desired state. + + desired := dc.Records + var err error + desired, err = handsoff(existing, desired, dc.Unmanaged, dc.UnmanagedUnsafe) // Handle UNMANAGED() + if err != nil { + return nil, err + } + + cc := NewCompareConfig(dc.Name, existing, desired, compFunc) + instructions := analyzeByRecordSet(cc) + return processPurge(instructions, !dc.KeepUnknown), nil +} + +// ByLabel takes two lists of records (existing and desired) and +// returns instructions for turning existing into desired. +// +// Use this with DNS providers whose API updates one label at a +// time. That is, updates are done by sending a list of DNS records +// to be served at a particular label, or the label itself is deleted. +// +// Examples include: +func ByLabel(existing models.Records, dc *models.DomainConfig, compFunc ComparableFunc) (ChangeList, error) { + // dc stores the desired state. + + desired := dc.Records + var err error + desired, err = handsoff(existing, desired, dc.Unmanaged, dc.UnmanagedUnsafe) // Handle UNMANAGED() + if err != nil { + return nil, err + } + + cc := NewCompareConfig(dc.Name, existing, desired, compFunc) + instructions := analyzeByLabel(cc) + return processPurge(instructions, !dc.KeepUnknown), nil +} + +// ByRecord takes two lists of records (existing and desired) and +// returns instructions for turning existing into desired. +// +// Use this with DNS providers whose API updates one record at a time. +// +// Examples include: INWX +func ByRecord(existing models.Records, dc *models.DomainConfig, compFunc ComparableFunc) (ChangeList, error) { + // dc stores the desired state. + + desired := dc.Records + var err error + desired, err = handsoff(existing, desired, dc.Unmanaged, dc.UnmanagedUnsafe) // Handle UNMANAGED() + if err != nil { + return nil, err + } + + cc := NewCompareConfig(dc.Name, existing, desired, compFunc) + instructions := analyzeByRecord(cc) + return processPurge(instructions, !dc.KeepUnknown), nil +} + +// ByZone takes two lists of records (existing and desired) and +// returns text one would output to users describing the change. +// +// Use this with DNS providers whose API updates the entire zone at a +// time. That is, to make any change (1 record or many) the entire DNS +// zone is uploaded. +// +// The user should see a list of changes as if individual records were +// updated. +// +// The caller of this function should: +// +// changed, msgs := diff2.ByZone(existing, desired, origin, nil +// fmt.Sprintf("CREATING ZONEFILE FOR THE FIRST TIME: dir/example.com.zone")) +// if changed { +// // output msgs +// // generate the zone using the "desired" records +// } +// +// Example providers include: BIND +func ByZone(existing models.Records, dc *models.DomainConfig, compFunc ComparableFunc) ([]string, bool, error) { + // dc stores the desired state. + + if len(existing) == 0 { + // Nothing previously existed. No need to output a list of individual changes. + return nil, true, nil + } + + desired := dc.Records + var err error + desired, err = handsoff(existing, desired, dc.Unmanaged, dc.UnmanagedUnsafe) // Handle UNMANAGED() + if err != nil { + return nil, false, err + } + + cc := NewCompareConfig(dc.Name, existing, desired, compFunc) + instructions := analyzeByRecord(cc) + instructions = processPurge(instructions, !dc.KeepUnknown) + return justMsgs(instructions), len(instructions) != 0, nil +} + +/* +nil, nil : +nil, nonzero : nil, true, nil +nonzero, nil : msgs, true, nil +nonzero, nonzero : + +existing: changes : return msgs, true, nil +existing: no changes : return nil, false, nil +not existing: no changes: return nil, false, nil +not existing: changes : return nil, true, nil +*/ + +func (c Change) String() string { + var buf bytes.Buffer + b := &buf + + fmt.Fprintf(b, "Change: verb=%v\n", c.Type) + fmt.Fprintf(b, " key=%v\n", c.Key) + if len(c.Old) != 0 { + fmt.Fprintf(b, " old=%v\n", c.Old) + } + if len(c.New) != 0 { + fmt.Fprintf(b, " new=%v\n", c.New) + } + fmt.Fprintf(b, " msg=%q\n", c.Msgs) + + return b.String() +} + +func (cl ChangeList) String() string { + var buf bytes.Buffer + b := &buf + + fmt.Fprintf(b, "ChangeList: len=%d\n", len(cl)) + for i, j := range cl { + fmt.Fprintf(b, "%02d: %s", i, j) + } + + return b.String() +} diff --git a/pkg/diff2/groupsort.go b/pkg/diff2/groupsort.go new file mode 100644 index 000000000..30dacbec0 --- /dev/null +++ b/pkg/diff2/groupsort.go @@ -0,0 +1,45 @@ +package diff2 + +import ( + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/StackExchange/dnscontrol/v3/pkg/prettyzone" +) + +type recset struct { + Key models.RecordKey + Recs []*models.RecordConfig +} + +func groupbyRSet(recs models.Records, origin string) []recset { + + if len(recs) == 0 { + return nil + } + + // Sort the NameFQDN to a consistent order. The actual sort methodology + // doesn't matter as long as equal values are adjacent. + // Use the PrettySort ordering so that the records are in a nice order. + pretty := prettyzone.PrettySort(recs, origin, 0, nil) + recs = pretty.Records + + var result []recset + var acc []*models.RecordConfig + + // Do the first element + prevkey := recs[0].Key() + acc = append(acc, recs[0]) + + for i := 1; i < len(recs); i++ { + curkey := recs[i].Key() + if prevkey == curkey { // A run of equal keys. + acc = append(acc, recs[i]) + } else { // New key. Append old data to result and start new acc. + result = append(result, recset{Key: prevkey, Recs: acc}) + acc = []*models.RecordConfig{recs[i]} + } + prevkey = curkey + } + result = append(result, recset{Key: prevkey, Recs: acc}) // The remainder + + return result +} diff --git a/pkg/diff2/groupsort_test.go b/pkg/diff2/groupsort_test.go new file mode 100644 index 000000000..07f806649 --- /dev/null +++ b/pkg/diff2/groupsort_test.go @@ -0,0 +1,45 @@ +package diff2 + +import ( + "reflect" + "testing" + + "github.com/StackExchange/dnscontrol/v3/models" +) + +func makeRec(label, rtype, content string) *models.RecordConfig { + origin := "f.com" + r := models.RecordConfig{} + r.SetLabel(label, origin) + r.PopulateFromString(rtype, content, origin) + return &r +} +func makeRecSet(recs ...*models.RecordConfig) *recset { + result := recset{} + result.Key = recs[0].Key() + result.Recs = append(result.Recs, recs...) + return &result +} +func Test_groupbyRSet(t *testing.T) { + + wwwa1 := makeRec("www", "A", "1.1.1.1") + wwwa2 := makeRec("www", "A", "2.2.2.2") + zzza1 := makeRec("zzz", "A", "1.1.0.0") + zzza2 := makeRec("zzz", "A", "2.2.0.0") + wwwmx1 := makeRec("www", "MX", "1 mx1.foo.com.") + wwwmx2 := makeRec("www", "MX", "2 mx2.foo.com.") + zzzmx1 := makeRec("zzz", "MX", "1 mx.foo.com.") + orig := models.Records{wwwa1, wwwa2, zzza1, zzza2, wwwmx1, wwwmx2, zzzmx1} + wantResult := []recset{ + *makeRecSet(wwwa1, wwwa2), + *makeRecSet(wwwmx1, wwwmx2), + *makeRecSet(zzza1, zzza2), + *makeRecSet(zzzmx1), + } + + t.Run("afew", func(t *testing.T) { + if gotResult := groupbyRSet(orig, "f.com"); !reflect.DeepEqual(gotResult, wantResult) { + t.Errorf("groupbyRSet() = %v, want %v", gotResult, wantResult) + } + }) +} diff --git a/pkg/diff2/highest.go b/pkg/diff2/highest.go new file mode 100644 index 000000000..8c1fbf44c --- /dev/null +++ b/pkg/diff2/highest.go @@ -0,0 +1,7 @@ +package diff2 + +// Return the highest valid index for an array. The equiv of len(s)-1, but with +// less likelihood that you'll commit an off-by-one error. +func highest[S ~[]T, T any](s S) int { + return len(s) - 1 +} diff --git a/pkg/diff2/min.go b/pkg/diff2/min.go new file mode 100644 index 000000000..402b27cc8 --- /dev/null +++ b/pkg/diff2/min.go @@ -0,0 +1,10 @@ +package diff2 + +import "golang.org/x/exp/constraints" + +func min[T constraints.Ordered](a, b T) T { + if a < b { + return a + } + return b +} diff --git a/pkg/diff2/nopurge.go b/pkg/diff2/nopurge.go new file mode 100644 index 000000000..bd41f4f83 --- /dev/null +++ b/pkg/diff2/nopurge.go @@ -0,0 +1,22 @@ +package diff2 + +func processPurge(instructions ChangeList, nopurge bool) ChangeList { + + if nopurge { + return instructions + } + + // TODO(tlim): This can probably be done without allocations but it + // works and I won't want to prematurely optimize. + + newinstructions := make(ChangeList, 0, len(instructions)) + for _, j := range instructions { + if j.Type == DELETE { + continue + } + newinstructions = append(newinstructions, j) + } + + return newinstructions + +} diff --git a/pkg/diff2/notes.txt b/pkg/diff2/notes.txt new file mode 100644 index 000000000..d0bd536b5 --- /dev/null +++ b/pkg/diff2/notes.txt @@ -0,0 +1,198 @@ +EXISTING: + laba A 1.2.3.4 [0] + laba MX 10 laba [1] + labc CNAME laba [2] + labe A 10.10.10.15 [3] + labe A 10.10.10.16 [4] + labe A 10.10.10.17 [5] + labe A 10.10.10.18 [6] + labg NS 10.10.10.15 [7] + labg NS 10.10.10.16 [8] + labg NS 10.10.10.17 [9] + labg NS 10.10.10.18 [10] + labh CNAME labd [11] + +DESIRED: + laba A 1.2.3.4 [0'] + laba A 1.2.3.5 [1'] + laba MX 20 labb [2'] + labe A 10.10.10.95 [3'] + labe A 10.10.10.96 [4'] + labe A 10.10.10.97 [5'] + labe A 10.10.10.98 [6'] + labf TXT "foo" [7'] + labg NS 10.10.10.10 [8'] + labg NS 10.10.10.15 [9'] + labg NS 10.10.10.16 [10'] + labg NS 10.10.10.97 [11'] + labh A 1.2.3.4 [12'] + +ByRRSet: + [] laba:A CHANGE NewSet: { [0], [1'] } (ByRecords needs: Old [0] ) + [] laba:MX CHANGE NewSet: { [2'] } (ByLabel needs: Old: [2]) + [] labc:CNAME DELETE Old: { [2 ] } + [] labe:A CHANGE NewSet: { [3'], [4'], [5'], [6'] } + [] labf:TXT CHANGE NewSet: { [7'] } + [] labg:NS CHANGE NewSet: { [7] [8] [8'] [11'] } + [] labh:CNAME DELETE Old { [11] } + [] labh:A CREATE NewSet: { [12'] } + +ByRecord: +CREATE [1'] +CHANGE [1] [2'] +DELETE [2] +CHANGE [3] [3'] +CHANGE [4] [4'] +CHANGE [5] [5'] +CHANGE [6] [6'] +CREATE [7'] +CREATE [8'] +CHANGE [10] [11'] +DELETE [11] +CREATE [12'] + + +ByLabel: (take ByRRSet gather all CHANGES) +laba CHANGE NewSet: { [0'], [1'], [2'] } +labc DELETE Old: { [2] } +labe CHANGE New: { [3'], [4'], [5'], [6'] } +labf CREATE New: { [7'] } +labg CHANGE NewSet: { [7] [8] [8'] [11'] } +labh DELETE Old { [11] } +labh CREATE NewSet: { [12'] } + + + + +By Record: + +rewrite as triples: FQDN+TYPE, TARGET, RC +byRecord: + group-by key=FQDN+TYPE, use targets to make add/change/delete for each record. + +byRSet: + group-by key=FQDN+TYPE, use targets to make add/change/delete for each record. + for each key: + if both have this key: + IF targets are the same, skip. + Else generate CHANGE for KEY: + New = Recs from desired. + Msgs = The msgs from targetdiff(e.Recs, d.Recs) + +byLabel: + group-by key=FQDN, use type+targets to make add/change/delete for each record. + + +rewrite as triples: FQDN {, TYPE, TARGET, RC + +type CompareConfig struct { + existing, desired models.Records + ldata: []LabelConfig +} + +type ByLabelConfig struct { + label string + tdata: []ByRTypeConfig +} + +type ByRTypeConfig struct { + rtype string + existing: []TargetConfig + desired: []TargetConfig + existingRecs: []*models.RecordConfig + desiredRecs: []*models.RecordConfig +} + +type TargetConfig struct { + compareable string + rec *model.RecordConfig +} + +func highest[S ~[]T, T any](s S) int { + return len(s) - 1 +} + +populate CompareConfig. +for rec := range desired { + label = FILL + rtype = FILL + comp = FILL + cc.labelMap[label] = &rc + cc.keyMap[key] = &rc + if not seen label: + append cc.ldata ByLabelConfig{} + labelIdx = last(cc.ldata) + if not seen key: + append cc.ldata[labelIdx].tdata ByRTypeConfig{} + rtIdx = last(cc.ldata[labelIdx].tdata) + cc.ldata[labelIdx].label = label + cc.ldata[labelIdx].tdata[rtIdx].rtype = rtype + cc.ldata[labelIdx].tdata[rtIdx].existing[append].comparable = comp + cc.ldata[labelIdx].tdata[rtIdx].existing[append].rec = &rc +} + +ByRSet: +func traverse(cc CompareConfig) { + for label := range cc.data { + for rtype := range label.data { + Msgs := genmsgs(rtype.existing, rtype.desired) + if no Msgs, continue + if len(rtype.existing) = 0 { + yield create(label, rtype, rtype.desiredRecs, Msgs) + } else if len(rtype.desired) = 0 { + yield delete(label, rtype, rtype.existingRecs, Msgs) + } else { // equal + yield change(label, rtype, rtype.desiredRecs, Msgs) + } + } +} + +byLabel: +func traverse(cc CompareConfig) { + for label := range cc.data { + initialize msgs, desiredRecords + anyExisting = false + for rtype := range label.data { + accumulate Msgs := genmsgs(rtype.existing, rtype.desired) + if Msgs (i.e. there were changes) { + accumulate AllDesired := rtype.desiredRecs + if len(rtype.existing) != 0 { + anyExisting = true + } + } + } + if there are Msgs: + if len(AllDesired) = 0 { + yield delete(label) + } else if countAllExisting == 0 { + yield create(label, AllDesired) + } else { + yield change(label, AllDesired) + } + } +} + +ByRecord: +func traverse(cc CompareConfig) { + for label := range cc.data { + for rtype := range label.data { + create, change, delete := difftargets(rtype.existing, rtype.desired) + yield creates, changes, deletes + } + } +} + +Byzone: +func traverse(cc CompareConfig) { + for label := range cc.data { + for rtype := range label.data { + Msgs := genmsgs(rtype.existing, rtype.desired) + accumulate Msgs + } + } + if len(accumMsgs) == 0 { + return nil, FirstMsg + } else { + return desired, Msgs + } +} \ No newline at end of file diff --git a/pkg/diff2/unmanaged.go b/pkg/diff2/unmanaged.go new file mode 100644 index 000000000..9401fafd6 --- /dev/null +++ b/pkg/diff2/unmanaged.go @@ -0,0 +1,129 @@ +package diff2 + +import ( + "fmt" + "strings" + + "github.com/gobwas/glob" + + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/StackExchange/dnscontrol/v3/pkg/printer" +) + +func handsoff( + existing, desired models.Records, + unmanaged []*models.UnmanagedConfig, + beSafe bool, +) (models.Records, error) { + + // What foreign items should we ignore? + foreign, err := manyQueries(existing, unmanaged) + if err != nil { + return nil, err + } + if len(foreign) != 0 { + printer.Printf("INFO: Foreign records being ignored: (%d)\n", len(foreign)) + for i, r := range foreign { + printer.Printf("- % 4d: %s %s %s\n", i, r.GetLabelFQDN(), r.Type, r.GetTargetRFC1035Quoted()) + } + } + + // What desired items might conflict? + conflicts, err := manyQueries(desired, unmanaged) + if err != nil { + return nil, err + } + if len(conflicts) != 0 { + level := "WARN" + if beSafe { + level = "ERROR" + } + printer.Printf("%s: dnsconfig.js records that overlap MANAGED: (%d)\n", level, len(conflicts)) + for i, r := range conflicts { + printer.Printf("- % 4d: %s %s %s\n", i, r.GetLabelFQDN(), r.Type, r.GetTargetRFC1035Quoted()) + } + if beSafe { + return nil, fmt.Errorf("ERROR: Unsafe to continue. Add DISABLE_UNMANAGED_SAFETY_CHECK to D() to override") + } + } + + // Add the foreign items to the desired list. + // (Rather than literally ignoring them, we just add them to the desired list + // and all the diffing algorithms become more simple.) + desired = append(desired, foreign...) + + return desired, nil +} + +func manyQueries(rcs models.Records, queries []*models.UnmanagedConfig) (result models.Records, err error) { + + for _, q := range queries { + + lab := q.Label + if lab == "" { + lab = "*" + } + glabel, err := glob.Compile(lab) + if err != nil { + return nil, err + } + + targ := q.Target + if targ == "" { + targ = "*" + } + gtarget, err := glob.Compile(targ) + if err != nil { + return nil, err + } + + hasRType := compileTypeGlob(q.RType) + + for _, rc := range rcs { + if match(rc, glabel, gtarget, hasRType) { + result = append(result, rc) + } + } + } + return result, nil +} + +func compileTypeGlob(g string) map[string]bool { + m := map[string]bool{} + for _, j := range strings.Split(g, ",") { + m[strings.TrimSpace(j)] = true + } + return m +} + +func match(rc *models.RecordConfig, glabel, gtarget glob.Glob, hasRType map[string]bool) bool { + printer.Printf("DEBUG: match(%v, %v, %v, %v)\n", rc.NameFQDN, glabel, gtarget, hasRType) + + // _ = glabel.Match(rc.NameFQDN) + // _ = matchType(rc.Type, hasRType) + // x := rc.GetTargetField() + // _ = gtarget.Match(x) + + if !glabel.Match(rc.NameFQDN) { + printer.Printf("DEBUG: REJECTED LABEL: %s:%v\n", rc.NameFQDN, glabel) + return false + } else if !matchType(rc.Type, hasRType) { + printer.Printf("DEBUG: REJECTED TYPE: %s:%v\n", rc.Type, hasRType) + return false + } else if gtarget == nil { + return true + } else if !gtarget.Match(rc.GetTargetField()) { + printer.Printf("DEBUG: REJECTED TARGET: %v:%v\n", rc.GetTargetField(), gtarget) + return false + } + return true +} + +func matchType(s string, hasRType map[string]bool) bool { + printer.Printf("DEBUG: matchType map=%v\n", hasRType) + if len(hasRType) == 0 { + return true + } + _, ok := hasRType[s] + return ok +} diff --git a/pkg/diff2/unmanaged_test.go b/pkg/diff2/unmanaged_test.go new file mode 100644 index 000000000..6d0fd38b9 --- /dev/null +++ b/pkg/diff2/unmanaged_test.go @@ -0,0 +1,147 @@ +package diff2 + +import ( + "testing" + + "github.com/StackExchange/dnscontrol/v3/models" + "github.com/gobwas/glob" +) + +var rmapNil map[string]bool +var rmapAMX = map[string]bool{ + "A": true, + "MX": true, +} +var rmapCNAME = map[string]bool{ + "CNAME": true, +} + +func Test_match(t *testing.T) { + + testRecLammaA1234 := makeRec("lamma", "A", "1.2.3.4") + + type args struct { + rc *models.RecordConfig + glabel glob.Glob + gtarget glob.Glob + hasRType map[string]bool + } + tests := []struct { + name string + args args + want bool + }{ + + { + name: "match3", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("lam*"), + hasRType: rmapAMX, + gtarget: glob.MustCompile("1.2.3.*"), + }, + want: true, + }, + + { + name: "match2", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("lam*"), + hasRType: rmapAMX, + gtarget: nil, + }, + want: true, + }, + + { + name: "match1", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("lam*"), + hasRType: rmapNil, + gtarget: nil, + }, + want: true, + }, + + { + name: "reject1", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("yyyy"), + hasRType: rmapAMX, + gtarget: glob.MustCompile("1.2.3.*"), + }, + want: false, + }, + + { + name: "reject2", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("lam*"), + hasRType: rmapCNAME, + gtarget: glob.MustCompile("1.2.3.*"), + }, + want: false, + }, + + { + name: "reject3", + args: args{ + rc: testRecLammaA1234, + glabel: glob.MustCompile("lam*"), + hasRType: rmapAMX, + gtarget: glob.MustCompile("zzzzz"), + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := match(tt.args.rc, tt.args.glabel, tt.args.gtarget, tt.args.hasRType); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_matchType(t *testing.T) { + type args struct { + s string + hasRType map[string]bool + } + tests := []struct { + name string + args args + want bool + }{ + + { + name: "matchCNAME", + args: args{"CNAME", rmapCNAME}, + want: true, + }, + + { + name: "rejectCNAME", + args: args{"MX", rmapCNAME}, + want: false, + }, + + { + name: "matchNIL", + args: args{"CNAME", rmapNil}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := matchType(tt.args.s, tt.args.hasRType); got != tt.want { + t.Errorf("matchType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/diff2/verb_string.go b/pkg/diff2/verb_string.go new file mode 100644 index 000000000..eab477f5b --- /dev/null +++ b/pkg/diff2/verb_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=Verb"; DO NOT EDIT. + +package diff2 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CREATE-1] + _ = x[CHANGE-2] + _ = x[DELETE-3] +} + +const _Verb_name = "CREATECHANGEDELETE" + +var _Verb_index = [...]uint8{0, 6, 12, 18} + +func (i Verb) String() string { + i -= 1 + if i < 0 || i >= Verb(len(_Verb_index)-1) { + return "Verb(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Verb_name[_Verb_index[i]:_Verb_index[i+1]] +} diff --git a/pkg/js/helpers.js b/pkg/js/helpers.js index e750a2a0e..46a16eefe 100644 --- a/pkg/js/helpers.js +++ b/pkg/js/helpers.js @@ -109,6 +109,7 @@ function newDomain(name, registrar) { nameservers: [], ignored_names: [], ignored_targets: [], + unmanaged: [], }; } @@ -610,6 +611,11 @@ function IGNORE_NAME(name, rTypes) { } return function (d) { d.ignored_names.push({ pattern: name, types: rTypes }); + d.unmanaged.push({ + label_pattern: name, + rType_pattern: rTypes, + target_pattern: '*', + }); }; } @@ -622,10 +628,14 @@ var IGNORE_NAME_DISABLE_SAFETY_CHECK = { // See https://github.com/StackExchange/dnscontrol/issues/1106 }; -// IGNORE_TARGET(target, rType) function IGNORE_TARGET(target, rType) { return function (d) { d.ignored_targets.push({ pattern: target, type: rType }); + d.unmanaged.push({ + label_pattern: '*', + rType_pattern: rType, + target_pattern: target, + }); }; } @@ -667,6 +677,34 @@ function AUTODNSSEC(d) { ); } +function UNMANAGED(label_pattern, rType_pattern, target_pattern) { + if (rType_pattern === undefined) { + rType_pattern = '*'; + } + if (rType_pattern === "") { + rType_pattern = '*'; + } + if (target_pattern === undefined) { + target_pattern = '*'; + } + return function (d) { + d.unmanaged.push({ + label_pattern: label_pattern, + rType_pattern: rType_pattern, + target_pattern: target_pattern, + }); + }; +} + +function DISABLE_UNMANAGED_SAFETY_CHECK(d) { + // This disables a safety check intended to prevent DNSControl and + // another system getting into a battle as they both try to update + // the same record over and over, back and forth. However, people + // kept asking for it so... caveat emptor! + // It only affects the current domain. + d.unmanaged_disable_safety_check = true; +} + /** * @deprecated */ diff --git a/pkg/js/js_test.go b/pkg/js/js_test.go index 23e09416f..1a19ea5f1 100644 --- a/pkg/js/js_test.go +++ b/pkg/js/js_test.go @@ -86,7 +86,7 @@ func TestParsedFiles(t *testing.T) { as := string(actualJSON) _, _ = es, as // When debugging, leave behind the actual result: - // os.WriteFile(expectedFile+".ACTUAL", []byte(es), 0644) + //os.WriteFile(expectedFile+".ACTUAL", []byte(as), 0644) testifyrequire.JSONEqf(t, es, as, "EXPECTING %q = \n```\n%s\n```", expectedFile, as) // For each domain, if there is a zone file, test against it: diff --git a/pkg/js/parse_tests/005-ignored-records.json b/pkg/js/parse_tests/005-ignored-records.json index 52d86f812..2fd8a0c95 100644 --- a/pkg/js/parse_tests/005-ignored-records.json +++ b/pkg/js/parse_tests/005-ignored-records.json @@ -1,8 +1,12 @@ { + "registrars": [], "dns_providers": [], "domains": [ { + "name": "foo.com", + "registrar": "none", "dnsProviders": {}, + "records": [], "ignored_names": [ { "pattern": "testignore", @@ -39,10 +43,48 @@ "type": "CNAME" } ], - "name": "foo.com", - "records": [], - "registrar": "none" + "unmanaged": [ + { + "label_pattern": "testignore", + "rType_pattern": "*", + "target_pattern": "*" + }, + { + "label_pattern": "testignore2", + "rType_pattern": "A", + "target_pattern": "*" + }, + { + "label_pattern": "testignore3", + "rType_pattern": "A, CNAME, TXT", + "target_pattern": "*" + }, + { + "label_pattern": "testignore4", + "rType_pattern": "*", + "target_pattern": "*" + }, + { + "label_pattern": "*", + "rType_pattern": "CNAME", + "target_pattern": "testtarget" + }, + { + "label_pattern": "legacyignore", + "rType_pattern": "*", + "target_pattern": "*" + }, + { + "label_pattern": "@", + "rType_pattern": "*", + "target_pattern": "*" + }, + { + "label_pattern": "*", + "rType_pattern": "CNAME", + "target_pattern": "@" + } + ] } - ], - "registrars": [] -} + ] +} \ No newline at end of file diff --git a/pkg/js/parse_tests/023-ignored-glob-records.json b/pkg/js/parse_tests/023-ignored-glob-records.json index c3d62197a..8ea43afdd 100644 --- a/pkg/js/parse_tests/023-ignored-glob-records.json +++ b/pkg/js/parse_tests/023-ignored-glob-records.json @@ -12,7 +12,14 @@ "pattern": "\\*.testignore", "types": "*" } + ], + "unmanaged": [ + { + "label_pattern": "\\*.testignore", + "rType_pattern": "*", + "target_pattern": "*" + } ] } ] -} +} \ No newline at end of file diff --git a/pkg/js/parse_tests/042-unmanaged.js b/pkg/js/parse_tests/042-unmanaged.js new file mode 100644 index 000000000..7c54b3853 --- /dev/null +++ b/pkg/js/parse_tests/042-unmanaged.js @@ -0,0 +1,6 @@ +D("foo.com", "none" + , UNMANAGED("one") + , UNMANAGED("two", "A, CNAME") + , UNMANAGED("three", "TXT", "findme") + , UNMANAGED("notype", "", "targglob") +); diff --git a/pkg/js/parse_tests/042-unmanaged.json b/pkg/js/parse_tests/042-unmanaged.json new file mode 100644 index 000000000..07b24018a --- /dev/null +++ b/pkg/js/parse_tests/042-unmanaged.json @@ -0,0 +1,34 @@ +{ + "registrars": [], + "dns_providers": [], + "domains": [ + { + "name": "foo.com", + "registrar": "none", + "dnsProviders": {}, + "records": [], + "unmanaged": [ + { + "label_pattern": "one", + "rType_pattern": "*", + "target_pattern": "*" + }, + { + "label_pattern": "two", + "rType_pattern": "A, CNAME", + "target_pattern": "*" + }, + { + "label_pattern": "three", + "rType_pattern": "TXT", + "target_pattern": "findme" + }, + { + "label_pattern": "notype", + "rType_pattern": "*", + "target_pattern": "targglob" + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/js/parse_tests/043-safety.js b/pkg/js/parse_tests/043-safety.js new file mode 100644 index 000000000..ba66804a6 --- /dev/null +++ b/pkg/js/parse_tests/043-safety.js @@ -0,0 +1,5 @@ +D("unsafe.com", "none" + , DISABLE_UNMANAGED_SAFETY_CHECK +); +D("safe.com", "none" +); diff --git a/pkg/js/parse_tests/043-safety.json b/pkg/js/parse_tests/043-safety.json new file mode 100644 index 000000000..e79433575 --- /dev/null +++ b/pkg/js/parse_tests/043-safety.json @@ -0,0 +1,19 @@ +{ + "registrars": [], + "dns_providers": [], + "domains": [ + { + "name": "unsafe.com", + "registrar": "none", + "dnsProviders": {}, + "records": [], + "unmanaged_disable_safety_check": true + }, + { + "name": "safe.com", + "registrar": "none", + "dnsProviders": {}, + "records": [] + } + ] +} \ No newline at end of file diff --git a/pkg/prettyzone/prettyzone_test.go b/pkg/prettyzone/prettyzone_test.go index e2cc9c1ad..c02789db7 100644 --- a/pkg/prettyzone/prettyzone_test.go +++ b/pkg/prettyzone/prettyzone_test.go @@ -484,11 +484,11 @@ func TestZoneLabelLess(t *testing.T) { } for _, test := range tests { - actual := zoneLabelLess(test.e1, test.e2) + actual := LabelLess(test.e1, test.e2) if test.expected != actual { t.Errorf("%v: expected (%v) got (%v)\n", test.e1, test.e2, actual) } - actual = zoneLabelLess(test.e2, test.e1) + actual = LabelLess(test.e2, test.e1) // The reverse should work too: var expected bool if test.e1 == test.e2 { diff --git a/pkg/prettyzone/sorting.go b/pkg/prettyzone/sorting.go index bc84d9202..66d423686 100644 --- a/pkg/prettyzone/sorting.go +++ b/pkg/prettyzone/sorting.go @@ -26,6 +26,8 @@ func (z *ZoneGenData) Less(i, j int) bool { a, b := z.Records[i], z.Records[j] // Sort by name. + + // If we are at the apex, use "@" in the sorting. compA, compB := a.NameFQDN, b.NameFQDN if compA != compB { if a.Name == "@" { @@ -34,7 +36,7 @@ func (z *ZoneGenData) Less(i, j int) bool { if b.Name == "@" { compB = "@" } - return zoneLabelLess(compA, compB) + return LabelLess(compA, compB) } // sub-sort by type @@ -103,7 +105,7 @@ func (z *ZoneGenData) Less(i, j int) bool { return a.String() < b.String() } -func zoneLabelLess(a, b string) bool { +func LabelLess(a, b string) bool { // Compare two zone labels for the purpose of sorting the RRs in a Zone. // If they are equal, we are done. All other code is simplified diff --git a/providers/autodns/autoDnsProvider.go b/providers/autodns/autoDnsProvider.go index d6f8a0b3a..cd962f199 100644 --- a/providers/autodns/autoDnsProvider.go +++ b/providers/autodns/autoDnsProvider.go @@ -185,6 +185,7 @@ func (api *autoDNSProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*mo // Insert Future diff2 version here. return corrections, nil + } // GetNameservers returns the nameservers for a domain. diff --git a/providers/axfrddns/axfrddnsProvider.go b/providers/axfrddns/axfrddnsProvider.go index 97943986b..2abf4e630 100644 --- a/providers/axfrddns/axfrddnsProvider.go +++ b/providers/axfrddns/axfrddnsProvider.go @@ -450,4 +450,5 @@ func (c *axfrddnsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*mod // Insert Future diff2 version here. return corrections, nil + } diff --git a/providers/azuredns/azureDnsProvider.go b/providers/azuredns/azureDnsProvider.go index 9d44e8fdf..58920f1f2 100644 --- a/providers/azuredns/azureDnsProvider.go +++ b/providers/azuredns/azureDnsProvider.go @@ -197,7 +197,7 @@ func (a *azurednsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*mod txtutil.SplitSingleLongTxt(dc.Records) // Autosplit long TXT records var corrections []*models.Correction - if !diff2.EnableDiff2 || true { // Remove "|| true" when diff2 version arrives + if !diff2.EnableDiff2 { differ := diff.New(dc) namesToUpdate, err := differ.ChangedGroups(existingRecords) @@ -330,7 +330,69 @@ func (a *azurednsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*mod return corrections, nil } - // Insert Future diff2 version here. + // Azure is a "ByRSet" API. + + instructions, err := diff2.ByLabel(existingRecords, dc, nil) + if err != nil { + return nil, err + } + + for _, inst := range instructions { + switch inst.Type { + + case diff2.CHANGE, diff2.CREATE: + var rrset *adns.RecordSet + var recordName string + var recordType adns.RecordType + if len(inst.Old) == 0 { // Create + rrset = &adns.RecordSet{Type: to.StringPtr(inst.Key.Type), Properties: &adns.RecordSetProperties{}} + recordType, _ = nativeToRecordType(to.StringPtr(inst.Key.Type)) + recordName = inst.Key.NameFQDN + } else { // Change + rrset = inst.Old[0].Original.(*adns.RecordSet) + recordType, _ = nativeToRecordType(to.StringPtr(*rrset.Type)) + recordName = *rrset.Name + } + // ^^^ this is broken and can probably be cleaned up significantly by + // someone that understands Azure's API. + + corrections = append(corrections, + &models.Correction{ + Msg: strings.Join(inst.Msgs, "\n"), + F: func() error { + ctx, cancel := context.WithTimeout(context.Background(), 6000*time.Second) + defer cancel() + _, err := a.recordsClient.CreateOrUpdate(ctx, *a.resourceGroup, zoneName, recordName, recordType, *rrset, nil) + if err != nil { + return err + } + return nil + }, + }) + + case diff2.DELETE: + fmt.Printf("DEBUG: azure inst=%s\n", inst) + rrset := inst.Old[0].Original.(*adns.RecordSet) + corrections = append(corrections, + &models.Correction{ + Msg: strings.Join(inst.Msgs, "\n"), + F: func() error { + ctx, cancel := context.WithTimeout(context.Background(), 6000*time.Second) + defer cancel() + rt, err := nativeToRecordType(rrset.Type) + if err != nil { + return err + } + _, err = a.recordsClient.Delete(ctx, *a.resourceGroup, zoneName, *rrset.Name, rt, nil) + if err != nil { + return err + } + return nil + }, + }) + } + + } return corrections, nil } @@ -370,7 +432,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig case "Microsoft.Network/dnszones/A": if set.Properties.ARecords != nil { for _, rec := range set.Properties.ARecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "A" _ = rc.SetTarget(*rec.IPv4Address) @@ -383,6 +445,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig AzureAlias: map[string]string{ "type": "A", }, + Original: set, } rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) _ = rc.SetTarget(*set.Properties.TargetResource.ID) @@ -391,7 +454,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig case "Microsoft.Network/dnszones/AAAA": if set.Properties.AaaaRecords != nil { for _, rec := range set.Properties.AaaaRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "AAAA" _ = rc.SetTarget(*rec.IPv6Address) @@ -404,6 +467,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig AzureAlias: map[string]string{ "type": "AAAA", }, + Original: set, } rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) _ = rc.SetTarget(*set.Properties.TargetResource.ID) @@ -411,7 +475,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/CNAME": if set.Properties.CnameRecord != nil { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "CNAME" _ = rc.SetTarget(*set.Properties.CnameRecord.Cname) @@ -423,6 +487,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig AzureAlias: map[string]string{ "type": "CNAME", }, + Original: set, } rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) _ = rc.SetTarget(*set.Properties.TargetResource.ID) @@ -430,7 +495,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/NS": for _, rec := range set.Properties.NsRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "NS" _ = rc.SetTarget(*rec.Nsdname) @@ -438,7 +503,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/PTR": for _, rec := range set.Properties.PtrRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "PTR" _ = rc.SetTarget(*rec.Ptrdname) @@ -446,14 +511,14 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/TXT": if len(set.Properties.TxtRecords) == 0 { // Empty String Record Parsing - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "TXT" _ = rc.SetTargetTXT("") results = append(results, rc) } else { for _, rec := range set.Properties.TxtRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "TXT" var txts []string @@ -466,7 +531,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/MX": for _, rec := range set.Properties.MxRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "MX" _ = rc.SetTargetMX(uint16(*rec.Preference), *rec.Exchange) @@ -474,7 +539,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/SRV": for _, rec := range set.Properties.SrvRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "SRV" _ = rc.SetTargetSRV(uint16(*rec.Priority), uint16(*rec.Weight), uint16(*rec.Port), *rec.Target) @@ -482,7 +547,7 @@ func nativeToRecords(set *adns.RecordSet, origin string) []*models.RecordConfig } case "Microsoft.Network/dnszones/CAA": for _, rec := range set.Properties.CaaRecords { - rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL)} + rc := &models.RecordConfig{TTL: uint32(*set.Properties.TTL), Original: set} rc.SetLabelFromFQDN(*set.Properties.Fqdn, origin) rc.Type = "CAA" _ = rc.SetTargetCAA(uint8(*rec.Flags), *rec.Tag, *rec.Value) diff --git a/providers/bind/bindProvider.go b/providers/bind/bindProvider.go index 096b0b28f..1d829b224 100644 --- a/providers/bind/bindProvider.go +++ b/providers/bind/bindProvider.go @@ -169,7 +169,7 @@ func (c *bindProvider) GetZoneRecords(domain string) (models.Records, error) { if os.IsNotExist(err) { // If the file doesn't exist, that's not an error. Just informational. c.zoneFileFound = false - fmt.Fprintf(os.Stderr, "File does not yet exist: %q\n", c.zonefile) + fmt.Fprintf(os.Stderr, "File does not yet exist: %q (will create)\n", c.zonefile) return nil, nil } if err != nil { @@ -244,8 +244,10 @@ func (c *bindProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models. models.PostProcessRecords(foundRecords) txtutil.SplitSingleLongTxt(dc.Records) // Autosplit long TXT records - var corrections []*models.Correction - if !diff2.EnableDiff2 || true { // Remove "|| true" when diff2 version arrives + changes := false + var msg string + + if !diff2.EnableDiff2 { differ := diff.New(dc) _, create, del, mod, err := differ.IncrementalDiff(foundRecords) @@ -255,7 +257,7 @@ func (c *bindProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models. buf := &bytes.Buffer{} // Print a list of changes. Generate an actual change that is the zone - changes := false + for _, i := range create { changes = true if c.zoneFileFound { @@ -275,48 +277,56 @@ func (c *bindProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models. } } - var msg string if c.zoneFileFound { msg = fmt.Sprintf("GENERATE_ZONEFILE: '%s'. Changes:\n%s", dc.Name, buf) } else { msg = fmt.Sprintf("GENERATE_ZONEFILE: '%s' (new file with %d records)\n", dc.Name, len(create)) } - if changes { + } else { - // We only change the serial number if there is a change. - desiredSoa.SoaSerial = nextSerial - - corrections = append(corrections, - &models.Correction{ - Msg: msg, - F: func() error { - printer.Printf("WRITING ZONEFILE: %v\n", c.zonefile) - zf, err := os.Create(c.zonefile) - if err != nil { - return fmt.Errorf("could not create zonefile: %w", err) - } - // Beware that if there are any fake types, then they will - // be commented out on write, but we don't reverse that when - // reading, so there will be a diff on every invocation. - err = prettyzone.WriteZoneFileRC(zf, dc.Records, dc.Name, 0, comments) - - if err != nil { - return fmt.Errorf("failed WriteZoneFile: %w", err) - } - err = zf.Close() - if err != nil { - return fmt.Errorf("closing: %w", err) - } - return nil - }, - }) + var msgs []string + msgs, changes, err = diff2.ByZone(foundRecords, dc, nil) + if err != nil { + return nil, err } + //fmt.Printf("DEBUG: BIND changes=%v\n", changes) + msg = strings.Join(msgs, "\n") - return corrections, nil } - // Insert Future diff2 version here. + var corrections []*models.Correction + //fmt.Printf("DEBUG: BIND changes=%v\n", changes) + if changes { + + // We only change the serial number if there is a change. + desiredSoa.SoaSerial = nextSerial + + corrections = append(corrections, + &models.Correction{ + Msg: msg, + F: func() error { + printer.Printf("WRITING ZONEFILE: %v\n", c.zonefile) + zf, err := os.Create(c.zonefile) + if err != nil { + return fmt.Errorf("could not create zonefile: %w", err) + } + // Beware that if there are any fake types, then they will + // be commented out on write, but we don't reverse that when + // reading, so there will be a diff on every invocation. + err = prettyzone.WriteZoneFileRC(zf, dc.Records, dc.Name, 0, comments) + + if err != nil { + return fmt.Errorf("failed WriteZoneFile: %w", err) + } + err = zf.Close() + if err != nil { + return fmt.Errorf("closing: %w", err) + } + return nil + }, + }) + } return corrections, nil } diff --git a/providers/cloudflare/cloudflareProvider.go b/providers/cloudflare/cloudflareProvider.go index 14194b8b4..3cd4933b6 100644 --- a/providers/cloudflare/cloudflareProvider.go +++ b/providers/cloudflare/cloudflareProvider.go @@ -326,6 +326,7 @@ func (c *cloudflareProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*m // Insert Future diff2 version here. return corrections, nil + } func checkNSModifications(dc *models.DomainConfig) { diff --git a/providers/cloudns/cloudnsProvider.go b/providers/cloudns/cloudnsProvider.go index bba381aab..6e556daf5 100644 --- a/providers/cloudns/cloudnsProvider.go +++ b/providers/cloudns/cloudnsProvider.go @@ -190,6 +190,7 @@ func (c *cloudnsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode // Insert Future diff2 version here. return corrections, nil + } // GetZoneRecords gets the records of a zone and returns them in RecordConfig format. diff --git a/providers/cscglobal/dns.go b/providers/cscglobal/dns.go index f74277527..faae4139b 100644 --- a/providers/cscglobal/dns.go +++ b/providers/cscglobal/dns.go @@ -194,6 +194,7 @@ func (client *providerClient) GenerateDomainCorrections(dc *models.DomainConfig, // Insert Future diff2 version here. return corrections, nil + } func makePurge(domainname string, cor diff.Correlation) zoneResourceRecordEdit { diff --git a/providers/desec/desecProvider.go b/providers/desec/desecProvider.go index 00008a534..eab37483a 100644 --- a/providers/desec/desecProvider.go +++ b/providers/desec/desecProvider.go @@ -248,5 +248,6 @@ func (c *desecProvider) GenerateDomainCorrections(dc *models.DomainConfig, exist // Insert Future diff2 version here. + // Insert Future diff2 version here. return corrections, nil } diff --git a/providers/gandiv5/gandi_v5Provider.go b/providers/gandiv5/gandi_v5Provider.go index fa9d43bbf..ad8226c78 100644 --- a/providers/gandiv5/gandi_v5Provider.go +++ b/providers/gandiv5/gandi_v5Provider.go @@ -230,7 +230,7 @@ func (client *gandiv5Provider) GenerateDomainCorrections(dc *models.DomainConfig txtutil.SplitSingleLongTxt(dc.Records) // Autosplit long TXT records var corrections []*models.Correction - if !diff2.EnableDiff2 || true { // Remove "|| true" when diff2 version arrives + if !diff2.EnableDiff2 { // diff existing vs. current. differ := diff.New(dc) @@ -305,6 +305,7 @@ func (client *gandiv5Provider) GenerateDomainCorrections(dc *models.DomainConfig // First time putting data on this label. Create it. // We have to create the label one rtype at a time. + ns := recordsToNative(desiredRecords[label], dc.Name) for _, n := range ns { msg := strings.Join(msgsForLabel[label], "\n") domain := dc.Name @@ -339,7 +340,83 @@ func (client *gandiv5Provider) GenerateDomainCorrections(dc *models.DomainConfig return corrections, nil } - // Insert Future diff2 version here. + g := gandi.NewLiveDNSClient(config.Config{ + APIKey: client.apikey, + SharingID: client.sharingid, + Debug: client.debug, + }) + + // Gandi is a "ByLabel" API with the odd exception that changes must be + // done one label:rtype at a time. + instructions, err := diff2.ByLabel(existing, dc, nil) + if err != nil { + return nil, err + } + for _, inst := range instructions { + switch inst.Type { + + case diff2.CREATE: + // We have to create the label one rtype at a time. + natives := recordsToNative(inst.New, dc.Name) + for _, n := range natives { + label := inst.Key.NameFQDN + rtype := n.RrsetType + domain := dc.Name + shortname := dnsutil.TrimDomainName(label, dc.Name) + ttl := n.RrsetTTL + values := n.RrsetValues + key := models.RecordKey{NameFQDN: label, Type: rtype} + msg := strings.Join(inst.MsgsByKey[key], "\n") + corrections = append(corrections, + &models.Correction{ + Msg: msg, + F: func() error { + res, err := g.CreateDomainRecord(domain, shortname, rtype, ttl, values) + if err != nil { + return fmt.Errorf("%+v: %w", res, err) + } + return nil + }, + }) + } + + case diff2.CHANGE: + msgs := strings.Join(inst.Msgs, "\n") + domain := dc.Name + label := inst.Key.NameFQDN + shortname := dnsutil.TrimDomainName(label, dc.Name) + ns := recordsToNative(inst.New, dc.Name) + corrections = append(corrections, + &models.Correction{ + Msg: msgs, + F: func() error { + res, err := g.UpdateDomainRecordsByName(domain, shortname, ns) + if err != nil { + return fmt.Errorf("%+v: %w", res, err) + } + return nil + }, + }) + + case diff2.DELETE: + msgs := strings.Join(inst.Msgs, "\n") + domain := dc.Name + label := inst.Key.NameFQDN + shortname := dnsutil.TrimDomainName(label, dc.Name) + corrections = append(corrections, + &models.Correction{ + Msg: msgs, + F: func() error { + err := g.DeleteDomainRecordsByName(domain, shortname) + if err != nil { + return err + } + return nil + }, + }) + } + + } return corrections, nil } diff --git a/providers/oracle/oracleProvider.go b/providers/oracle/oracleProvider.go index 95b775185..8e64ce053 100644 --- a/providers/oracle/oracleProvider.go +++ b/providers/oracle/oracleProvider.go @@ -257,8 +257,6 @@ func (o *oracleProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*model for any size zone. */ - corrections := []*models.Correction{} - if len(create) > 0 { createRecords := models.Records{} desc := "" diff --git a/providers/ovh/ovhProvider.go b/providers/ovh/ovhProvider.go index 0379744b0..af4f541b7 100644 --- a/providers/ovh/ovhProvider.go +++ b/providers/ovh/ovhProvider.go @@ -138,8 +138,6 @@ func (c *ovhProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.C return nil, err } - corrections := []*models.Correction{} - for _, del := range delete { rec := del.Existing.Original.(*Record) corrections = append(corrections, &models.Correction{