From 7a4dca5ad5a1fcfaf51e559be1972af8757a59dc Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Thu, 1 Feb 2018 11:45:53 -0500 Subject: [PATCH] Refactor: Prelink providers to domains (#305) --- commands/commands.go | 66 +++++++++++++++++++++++++++---- commands/createDomains.go | 15 ++----- commands/previewPush.go | 71 +++++++++++++++++++--------------- models/dns.go | 35 ++++++++++++++--- models/provider.go | 27 +++++++++++++ pkg/nameservers/nameservers.go | 14 +++---- pkg/normalize/validate.go | 34 +++++----------- pkg/normalize/validate_test.go | 8 ++-- providers/providers.go | 42 +++----------------- 9 files changed, 181 insertions(+), 131 deletions(-) create mode 100644 models/provider.go diff --git a/commands/commands.go b/commands/commands.go index 1a77ec2b4..0af52a741 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -77,7 +77,7 @@ func (args *GetDNSConfigArgs) flags() []cli.Flag { ) } -// GetDNSConfig reads the json-formatted IR file. +// GetDNSConfig reads the json-formatted IR file. Or executes javascript. All depending on flags provided. func GetDNSConfig(args GetDNSConfigArgs) (*models.DNSConfig, error) { if args.JSONFile != "" { f, err := os.Open(args.JSONFile) @@ -90,9 +90,59 @@ func GetDNSConfig(args GetDNSConfigArgs) (*models.DNSConfig, error) { if err = dec.Decode(cfg); err != nil { return nil, err } - return cfg, nil + return preloadProviders(cfg, nil) } - return ExecuteDSL(args.ExecuteDSLArgs) + return preloadProviders(ExecuteDSL(args.ExecuteDSLArgs)) +} + +// the json only contains provider names inside domains. This denormalizes the data for more +// convenient access patterns. Does everything we need to prepare for the validation phase, but +// cannot do anything that requires the credentials file yet. +func preloadProviders(cfg *models.DNSConfig, err error) (*models.DNSConfig, error) { + if err != nil { + return cfg, err + } + //build name to type maps + cfg.RegistrarsByName = map[string]*models.RegistrarConfig{} + cfg.DNSProvidersByName = map[string]*models.DNSProviderConfig{} + for _, reg := range cfg.Registrars { + cfg.RegistrarsByName[reg.Name] = reg + } + for _, p := range cfg.DNSProviders { + cfg.DNSProvidersByName[p.Name] = p + } + // make registrar and dns provider shims. Include name, type, and other metadata, but can't inatantiate + // driver until we load creds in later + for _, d := range cfg.Domains { + reg, ok := cfg.RegistrarsByName[d.RegistrarName] + if !ok { + return nil, fmt.Errorf("Registrar named %s expected for %s, but never registered", d.RegistrarName, d.Name) + } + d.RegistrarInstance = &models.RegistrarInstance{ + ProviderBase: models.ProviderBase{ + Name: reg.Name, + ProviderType: reg.Type, + }, + } + for pName, n := range d.DNSProviderNames { + prov, ok := cfg.DNSProvidersByName[pName] + if !ok { + return nil, fmt.Errorf("DNS Provider named %s expected for %s, but never registered", pName, d.Name) + } + d.DNSProviderInstances = append(d.DNSProviderInstances, &models.DNSProviderInstance{ + ProviderBase: models.ProviderBase{ + Name: pName, + ProviderType: prov.Type, + }, + NumberOfNameservers: n, + }) + } + // sort so everything is deterministic + sort.Slice(d.DNSProviderInstances, func(i, j int) bool { + return d.DNSProviderInstances[i].Name < d.DNSProviderInstances[j].Name + }) + } + return cfg, nil } // ExecuteDSLArgs are used anytime we need to read and execute dnscontrol DSL @@ -185,20 +235,20 @@ func (args *FilterArgs) flags() []cli.Flag { } } -func (args *FilterArgs) shouldRunProvider(p string, dc *models.DomainConfig, nonDefaultProviders []string) bool { +func (args *FilterArgs) shouldRunProvider(name string, dc *models.DomainConfig) bool { if args.Providers == "all" { return true } if args.Providers == "" { - for _, pr := range nonDefaultProviders { - if pr == p { - return false + for _, pri := range dc.DNSProviderInstances { + if pri.Name == name { + return pri.IsDefault } } return true } for _, prov := range strings.Split(args.Providers, ",") { - if prov == p { + if prov == name { return true } } diff --git a/commands/createDomains.go b/commands/createDomains.go index 586a813c7..d0ef3cb17 100644 --- a/commands/createDomains.go +++ b/commands/createDomains.go @@ -2,7 +2,6 @@ package commands import ( "fmt" - "log" "github.com/StackExchange/dnscontrol/providers" "github.com/urfave/cli" @@ -38,21 +37,15 @@ func CreateDomains(args CreateDomainsArgs) error { if err != nil { return err } - registrars, dnsProviders, _, _, err := InitializeProviders(args.CredsFile, cfg, false) + _, err = InitializeProviders(args.CredsFile, cfg, false) if err != nil { return err } - fmt.Printf("Initialized %d registrars and %d dns service providers.\n", len(registrars), len(dnsProviders)) for _, domain := range cfg.Domains { fmt.Println("*** ", domain.Name) - for prov := range domain.DNSProviders { - dsp, ok := dnsProviders[prov] - if !ok { - log.Fatalf("DSP %s not declared.", prov) - } - if creator, ok := dsp.(providers.DomainCreator); ok { - fmt.Println(" -", prov) - // TODO: maybe return bool if it did anything. + for _, provider := range domain.DNSProviderInstances { + if creator, ok := provider.Driver.(providers.DomainCreator); ok { + fmt.Println(" -", provider.Name) err := creator.EnsureDomainExists(domain.Name) if err != nil { fmt.Printf("Error creating domain: %s\n", err) diff --git a/commands/previewPush.go b/commands/previewPush.go index e203c7d5e..3583c2ff4 100644 --- a/commands/previewPush.go +++ b/commands/previewPush.go @@ -96,11 +96,11 @@ func run(args PreviewArgs, push bool, interactive bool, out printer.CLI) error { if PrintValidationErrors(errs) { return fmt.Errorf("Exiting due to validation errors") } - registrars, dnsProviders, nonDefaultProviders, notifier, err := InitializeProviders(args.CredsFile, cfg, args.Notify) + // TODO: + notifier, err := InitializeProviders(args.CredsFile, cfg, args.Notify) if err != nil { return err } - out.Debugf("Initialized %d registrars and %d dns service providers.\n", len(registrars), len(dnsProviders)) anyErrors := false totalCorrections := 0 DomainLoop: @@ -109,45 +109,36 @@ DomainLoop: continue } out.StartDomain(domain.Name) - nsList, err := nameservers.DetermineNameservers(domain, 0, dnsProviders) + nsList, err := nameservers.DetermineNameservers(domain) if err != nil { return err } domain.Nameservers = nsList nameservers.AddNSRecords(domain) - for prov := range domain.DNSProviders { + for _, provider := range domain.DNSProviderInstances { dc, err := domain.Copy() if err != nil { return err } - shouldrun := args.shouldRunProvider(prov, dc, nonDefaultProviders) - out.StartDNSProvider(prov, !shouldrun) + shouldrun := args.shouldRunProvider(provider.Name, dc) + out.StartDNSProvider(provider.Name, !shouldrun) if !shouldrun { continue } - // TODO: make provider discovery like this a validate-time operation - dsp, ok := dnsProviders[prov] - if !ok { - log.Fatalf("DSP %s not declared.", prov) - } - corrections, err := dsp.GetDomainCorrections(dc) + corrections, err := provider.Driver.GetDomainCorrections(dc) out.EndProvider(len(corrections), err) if err != nil { anyErrors = true continue DomainLoop } totalCorrections += len(corrections) - anyErrors = printOrRunCorrections(domain.Name, prov, corrections, out, push, interactive, notifier) || anyErrors + anyErrors = printOrRunCorrections(domain.Name, provider.Name, corrections, out, push, interactive, notifier) || anyErrors } - run := args.shouldRunProvider(domain.Registrar, domain, nonDefaultProviders) - out.StartRegistrar(domain.Registrar, !run) + run := args.shouldRunProvider(domain.RegistrarName, domain) + out.StartRegistrar(domain.RegistrarName, !run) if !run { continue } - reg, ok := registrars[domain.Registrar] - if !ok { - log.Fatalf("Registrar %s not declared.", reg) - } if len(domain.Nameservers) == 0 && domain.Metadata["no_ns"] != "true" { out.Warnf("No nameservers declared; skipping registrar. Add {no_ns:'true'} to force.\n") continue @@ -156,14 +147,14 @@ DomainLoop: if err != nil { log.Fatal(err) } - corrections, err := reg.GetRegistrarCorrections(dc) + corrections, err := domain.RegistrarInstance.Driver.GetRegistrarCorrections(dc) out.EndProvider(len(corrections), err) if err != nil { anyErrors = true continue } totalCorrections += len(corrections) - anyErrors = printOrRunCorrections(domain.Name, domain.Registrar, corrections, out, push, interactive, notifier) || anyErrors + anyErrors = printOrRunCorrections(domain.Name, domain.RegistrarName, corrections, out, push, interactive, notifier) || anyErrors } if os.Getenv("TEAMCITY_VERSION") != "" { fmt.Fprintf(os.Stderr, "##teamcity[buildStatus status='SUCCESS' text='%d corrections']", totalCorrections) @@ -178,7 +169,7 @@ DomainLoop: // InitializeProviders takes a creds file path and a DNSConfig object. Creates all providers with the proper types, and returns them. // nonDefaultProviders is a list of providers that should not be run unless explicitly asked for by flags. -func InitializeProviders(credsFile string, cfg *models.DNSConfig, notifyFlag bool) (registrars map[string]providers.Registrar, dnsProviders map[string]providers.DNSServiceProvider, nonDefaultProviders []string, notify notifications.Notifier, err error) { +func InitializeProviders(credsFile string, cfg *models.DNSConfig, notifyFlag bool) (notify notifications.Notifier, err error) { var providerConfigs map[string]map[string]string var notificationCfg map[string]string defer func() { @@ -191,21 +182,39 @@ func InitializeProviders(credsFile string, cfg *models.DNSConfig, notifyFlag boo if notifyFlag { notificationCfg = providerConfigs["notifications"] } - nonDefaultProviders = []string{} + isNonDefault := map[string]bool{} for name, vals := range providerConfigs { // add "_exclude_from_defaults":"true" to a provider to exclude it from being run unless // -providers=all or -providers=name if vals["_exclude_from_defaults"] == "true" { - nonDefaultProviders = append(nonDefaultProviders, name) + isNonDefault[name] = true } } - registrars, err = providers.CreateRegistrars(cfg, providerConfigs) - if err != nil { - return - } - dnsProviders, err = providers.CreateDsps(cfg, providerConfigs) - if err != nil { - return + registrars := map[string]providers.Registrar{} + dnsProviders := map[string]providers.DNSServiceProvider{} + for _, d := range cfg.Domains { + if registrars[d.RegistrarName] == nil { + rCfg := cfg.RegistrarsByName[d.RegistrarName] + r, err := providers.CreateRegistrar(rCfg.Type, providerConfigs[d.RegistrarName]) + if err != nil { + return nil, err + } + registrars[d.RegistrarName] = r + } + d.RegistrarInstance.Driver = registrars[d.RegistrarName] + d.RegistrarInstance.IsDefault = !isNonDefault[d.RegistrarName] + for _, pInst := range d.DNSProviderInstances { + if dnsProviders[pInst.Name] == nil { + dCfg := cfg.DNSProvidersByName[pInst.Name] + prov, err := providers.CreateDNSProvider(dCfg.Type, providerConfigs[dCfg.Name], dCfg.Metadata) + if err != nil { + return nil, err + } + dnsProviders[pInst.Name] = prov + } + pInst.Driver = dnsProviders[pInst.Name] + pInst.IsDefault = !isNonDefault[pInst.Name] + } } return } diff --git a/models/dns.go b/models/dns.go index b026884cc..9d7397001 100644 --- a/models/dns.go +++ b/models/dns.go @@ -21,9 +21,11 @@ const DefaultTTL = uint32(300) // DNSConfig describes the desired DNS configuration, usually loaded from dnsconfig.js. type DNSConfig struct { - Registrars []*RegistrarConfig `json:"registrars"` - DNSProviders []*DNSProviderConfig `json:"dns_providers"` - Domains []*DomainConfig `json:"domains"` + Registrars []*RegistrarConfig `json:"registrars"` + DNSProviders []*DNSProviderConfig `json:"dns_providers"` + Domains []*DomainConfig `json:"domains"` + RegistrarsByName map[string]*RegistrarConfig `json:"-"` + DNSProvidersByName map[string]*DNSProviderConfig `json:"-"` } // FindDomain returns the *DomainConfig for domain query in config. @@ -348,20 +350,41 @@ func StringsToNameservers(nss []string) []*Nameserver { // DomainConfig describes a DNS domain (tecnically a DNS zone). type DomainConfig struct { - Name string `json:"name"` // NO trailing "." - Registrar string `json:"registrar"` - DNSProviders map[string]int `json:"dnsProviders"` + Name string `json:"name"` // NO trailing "." + RegistrarName string `json:"registrar"` + DNSProviderNames map[string]int `json:"dnsProviders"` + Metadata map[string]string `json:"meta,omitempty"` Records Records `json:"records"` Nameservers []*Nameserver `json:"nameservers,omitempty"` KeepUnknown bool `json:"keepunknown,omitempty"` IgnoredLabels []string `json:"ignored_labels,omitempty"` + + // These fields contain instantiated provider instances once everything is linked up. + // This linking is in two phases: + // 1. Metadata (name/type) is availible just from the dnsconfig. Validation can use that. + // 2. Final driver instances are loaded after we load credentials. Any actual provider interaction requires that. + RegistrarInstance *RegistrarInstance `json:"-"` + DNSProviderInstances []*DNSProviderInstance `json:"-"` } // Copy returns a deep copy of the DomainConfig. func (dc *DomainConfig) Copy() (*DomainConfig, error) { newDc := &DomainConfig{} + // provider instances are interfaces that gob hates if you don't register them. + // and the specific types are not gob encodable since nothing is exported. + // should find a better solution for this now. + // + // current strategy: remove everything, gob copy it. Then set both to stored copy. + reg := dc.RegistrarInstance + dnsps := dc.DNSProviderInstances + dc.RegistrarInstance = nil + dc.DNSProviderInstances = nil err := copyObj(dc, newDc) + dc.RegistrarInstance = reg + newDc.RegistrarInstance = reg + dc.DNSProviderInstances = dnsps + newDc.DNSProviderInstances = dnsps return newDc, err } diff --git a/models/provider.go b/models/provider.go new file mode 100644 index 000000000..b71110802 --- /dev/null +++ b/models/provider.go @@ -0,0 +1,27 @@ +package models + +type DNSProvider interface { + GetNameservers(domain string) ([]*Nameserver, error) + GetDomainCorrections(dc *DomainConfig) ([]*Correction, error) +} + +type Registrar interface { + GetRegistrarCorrections(dc *DomainConfig) ([]*Correction, error) +} + +type ProviderBase struct { + Name string + IsDefault bool + ProviderType string +} + +type RegistrarInstance struct { + ProviderBase + Driver Registrar +} + +type DNSProviderInstance struct { + ProviderBase + Driver DNSProvider + NumberOfNameservers int +} diff --git a/pkg/nameservers/nameservers.go b/pkg/nameservers/nameservers.go index 7d6a768b9..e4013f759 100644 --- a/pkg/nameservers/nameservers.go +++ b/pkg/nameservers/nameservers.go @@ -8,26 +8,22 @@ import ( "strconv" "github.com/StackExchange/dnscontrol/models" - "github.com/StackExchange/dnscontrol/providers" "github.com/miekg/dns/dnsutil" ) // DetermineNameservers will find all nameservers we should use for a domain. It follows the following rules: // 1. All explicitly defined NAMESERVER records will be used. // 2. Each DSP declares how many nameservers to use. Default is all. 0 indicates to use none. -func DetermineNameservers(dc *models.DomainConfig, maxNS int, dsps map[string]providers.DNSServiceProvider) ([]*models.Nameserver, error) { +func DetermineNameservers(dc *models.DomainConfig) ([]*models.Nameserver, error) { // always take explicit ns := dc.Nameservers - for dsp, n := range dc.DNSProviders { + for _, dnsProvider := range dc.DNSProviderInstances { + n := dnsProvider.NumberOfNameservers if n == 0 { continue } - fmt.Printf("----- Getting nameservers from: %s\n", dsp) - p, ok := dsps[dsp] - if !ok { - return nil, fmt.Errorf("DNS provider %s not declared", dsp) - } - nss, err := p.GetNameservers(dc.Name) + fmt.Printf("----- Getting nameservers from: %s\n", dnsProvider.Name) + nss, err := dnsProvider.Driver.GetNameservers(dc.Name) if err != nil { return nil, err } diff --git a/pkg/normalize/validate.go b/pkg/normalize/validate.go index 6ef2b44b9..a50531940 100644 --- a/pkg/normalize/validate.go +++ b/pkg/normalize/validate.go @@ -246,30 +246,19 @@ type Warning struct { // NormalizeAndValidateConfig performs and normalization and/or validation of the IR. func NormalizeAndValidateConfig(config *models.DNSConfig) (errs []error) { - ptypeMap := map[string]string{} - for _, p := range config.DNSProviders { - ptypeMap[p.Name] = p.Type - } - for _, domain := range config.Domains { pTypes := []string{} txtMultiDissenters := []string{} - for p := range domain.DNSProviders { - pType, ok := ptypeMap[p] - if !ok { - errs = append(errs, fmt.Errorf("%s uses undefined DNS provider %s", domain.Name, p)) - } else { - pTypes = append(pTypes, pType) - } - + for _, provider := range domain.DNSProviderInstances { + pType := provider.ProviderType // If NO_PURGE is in use, make sure this *isn't* a provider that *doesn't* support NO_PURGE. if domain.KeepUnknown && providers.ProviderHasCabability(pType, providers.CantUseNOPURGE) { - errs = append(errs, fmt.Errorf("%s uses NO_PURGE which is not supported by %s(%s)", domain.Name, p, pType)) + errs = append(errs, fmt.Errorf("%s uses NO_PURGE which is not supported by %s(%s)", domain.Name, provider.Name, pType)) } // Record if any providers do not support TXTMulti: if !providers.ProviderHasCabability(pType, providers.CanUseTXTMulti) { - txtMultiDissenters = append(txtMultiDissenters, p) + txtMultiDissenters = append(txtMultiDissenters, provider.Name) } } @@ -374,7 +363,7 @@ func NormalizeAndValidateConfig(config *models.DNSConfig) (errs []error) { // Check that if any aliases / ptr / etc.. are used in a domain, every provider for that domain supports them for _, d := range config.Domains { - err := checkProviderCapabilities(d, config.DNSProviders) + err := checkProviderCapabilities(d) if err != nil { errs = append(errs, err) } @@ -401,7 +390,7 @@ func checkCNAMEs(dc *models.DomainConfig) (errs []error) { return } -func checkProviderCapabilities(dc *models.DomainConfig, pList []*models.DNSProviderConfig) error { +func checkProviderCapabilities(dc *models.DomainConfig) error { types := []struct { rType string cap providers.Capability @@ -423,14 +412,9 @@ func checkProviderCapabilities(dc *models.DomainConfig, pList []*models.DNSProvi if !hasAny { continue } - for pName := range dc.DNSProviders { - for _, p := range pList { - if p.Name == pName { - if !providers.ProviderHasCabability(p.Type, ty.cap) { - return fmt.Errorf("Domain %s uses %s records, but DNS provider type %s does not support them", dc.Name, ty.rType, p.Type) - } - break - } + for _, provider := range dc.DNSProviderInstances { + if !providers.ProviderHasCabability(provider.ProviderType, ty.cap) { + return fmt.Errorf("Domain %s uses %s records, but DNS provider type %s does not support them", dc.Name, ty.rType, provider.ProviderType) } } } diff --git a/pkg/normalize/validate_test.go b/pkg/normalize/validate_test.go index 5c46839e1..96064f0b4 100644 --- a/pkg/normalize/validate_test.go +++ b/pkg/normalize/validate_test.go @@ -193,8 +193,8 @@ func TestCAAValidation(t *testing.T) { config := &models.DNSConfig{ Domains: []*models.DomainConfig{ { - Name: "example.com", - Registrar: "BIND", + Name: "example.com", + RegistrarName: "BIND", Records: []*models.RecordConfig{ {Name: "@", Type: "CAA", CaaTag: "invalid", Target: "example.com"}, }, @@ -211,8 +211,8 @@ func TestTLSAValidation(t *testing.T) { config := &models.DNSConfig{ Domains: []*models.DomainConfig{ { - Name: "_443._tcp.example.com", - Registrar: "BIND", + Name: "_443._tcp.example.com", + RegistrarName: "BIND", Records: []*models.RecordConfig{ {Name: "_443._tcp", Type: "TLSA", TlsaUsage: 4, TlsaSelector: 1, TlsaMatchingType: 1, Target: "abcdef0"}, }, diff --git a/providers/providers.go b/providers/providers.go index f923f85a1..893606cda 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -10,13 +10,12 @@ import ( // Registrar is an interface for a domain registrar. It can return a list of needed corrections to be applied in the future. type Registrar interface { - GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.Correction, error) + models.Registrar } // DNSServiceProvider is able to generate a set of corrections that need to be made to correct records for a domain type DNSServiceProvider interface { - GetNameservers(domain string) ([]*models.Nameserver, error) - GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) + models.DNSProvider } // DomainCreator should be implemented by providers that have the ability to add domains to an account. the create-domains command @@ -55,7 +54,8 @@ func RegisterDomainServiceProviderType(name string, init DspInitializer, pm ...P unwrapProviderCapabilities(name, pm) } -func createRegistrar(rType string, config map[string]string) (Registrar, error) { +// CreateRegistrar initializes a registrar instance from given credentials. +func CreateRegistrar(rType string, config map[string]string) (Registrar, error) { initer, ok := RegistrarTypes[rType] if !ok { return nil, fmt.Errorf("registrar type %s not declared", rType) @@ -63,7 +63,7 @@ func createRegistrar(rType string, config map[string]string) (Registrar, error) return initer(config) } -// CreateDNSProvider returnsa DSP's initializer. +// CreateDNSProvider initializes a dns provider instance from given credentials. func CreateDNSProvider(dType string, config map[string]string, meta json.RawMessage) (DNSServiceProvider, error) { initer, ok := DNSProviderTypes[dType] if !ok { @@ -72,38 +72,6 @@ func CreateDNSProvider(dType string, config map[string]string, meta json.RawMess return initer(config, meta) } -// CreateRegistrars will load all registrars from the dns config, and create instances of the correct type using data from -// the provider config to load relevant keys and options. -func CreateRegistrars(d *models.DNSConfig, providerConfigs map[string]map[string]string) (map[string]Registrar, error) { - regs := map[string]Registrar{} - for _, reg := range d.Registrars { - rawMsg, ok := providerConfigs[reg.Name] - if !ok && reg.Type != "NONE" { - return nil, fmt.Errorf("Registrar %s not listed in creds.json file", reg.Name) - } - registrar, err := createRegistrar(reg.Type, rawMsg) - if err != nil { - return nil, fmt.Errorf("Creating %s registrar: %s", reg.Name, err) - } - regs[reg.Name] = registrar - } - return regs, nil -} - -// CreateDsps creates a DSP. -func CreateDsps(d *models.DNSConfig, providerConfigs map[string]map[string]string) (map[string]DNSServiceProvider, error) { - dsps := map[string]DNSServiceProvider{} - for _, dsp := range d.DNSProviders { - vals := providerConfigs[dsp.Name] - provider, err := CreateDNSProvider(dsp.Type, vals, dsp.Metadata) - if err != nil { - return nil, fmt.Errorf("Creating %s dns provider: %s", dsp.Name, err) - } - dsps[dsp.Name] = provider - } - return dsps, nil -} - // None is a basic provider type that does absolutely nothing. Can be useful as a placeholder for third parties or unimplemented providers. type None struct{}