Update bind docs and Getting Started (#53)

* intitial work for bind refactor

* relax requirement that creds.json exists.

* Updating bind docs.
Fixes #48

* typo

* can exclude provider from default set in creds.json

* Add bind to integration tests. Fix for IDNs.

docs/_providers/bind.md

@@ -6,4 +6,31 @@ jsId: BIND
 # Bind Provider
 
 This provider simply maintains a directory with a collection of .zone files. We currently copy zone files to our production servers and restart bind via 
-a script external to DNSControl.
+a script external to DNSControl.
+
+## Configuration
+
+The BIND provider does not require anything in `creds.json`. It does accept some (optional) metadata via your dns config when you create the provider:
+
+{% highlight javascript %}
+var bind = NewDnsProvider('bind', 'BIND', {
+  'default_soa': {
+    'master': 'ns1.mydomain.com.',
+    'mbox': 'sysadmin.mydomain.com.',
+    'refresh': 3600,
+    'retry': 600,
+    'expire': 604800,
+    'minttl': 1440,
+  },
+  'default_ns': [
+        'ns1.mydomain.com.',
+        'ns2.mydomain.com.',
+        'ns3.mydomain.com.',
+        'ns4.mydomain.com.'
+  ]
+})
+{% endhighlight %}
+
+If you need to customize your SOA or NS records, you can do it with this setup.
+
+You can also provide a `-bindtree=directoryName` flag to change where the provider will look for and create zone files. The default is the `zones` directory where dnscontrol is run.

docs/getting-started.md

@@ -13,30 +13,19 @@ You can either download the latest [github release](https://github.com/StackExch
 
 The first file you will need is a javascript file to describe your domains.
 Individual providers will vary slightly. See [the provider docs]({{site.github.url}}/provider-list) for specifics.
-For this example we will use a domain registered with name.com, using their basic dns hosting.
+For this example we will use a single "BIND" provider that will generate zone files on our local file system.
 The default name is `dnsconfig.js`:
 
 {% highlight js %}
-var registrar = NewRegistrar("name.com",NAMEDOTCOM);
-var namecom = NewDnsProvider("name.com",NAMEDOTCOM);
+var registrar = NewRegistrar("none","NONE"); // no registrar
+var bind = NewDnsProvider("bind","BIND");
 
-D("example.com", registrar, DnsProvider(namecom),
+D("example.com", registrar, DnsProvider(bind),
   A("@", "1.2.3.4")
 );
 {%endhighlight%}
 
-The second file is a json document to hold your api credentials. By default we use `providers.json`:
-
-{% highlight json %}
-{
-  "name.com":{
-    "apikey": "yourApiKeyFromName.com-klasjdkljasdlk235235235235",
-    "apiuser": "yourUsername"
-  }
-}
-{%endhighlight%}
-
-You may modify these files to match your particular providers and domains. See [the javascript docs]({{site.github.url}}/js) for more details.
+You may modify this files to match your particular providers and domains. See [the javascript docs]({{site.github.url}}/js) and  [the provider docs]({{site.github.url}}/provider-list) for more details. If you are using other providers, you will likely need to make a `creds.json` file with api tokens and other account information.
 
 ## 3. Run `dnscontrol preview`
 
@@ -44,4 +33,4 @@ This will print out a list of "corrections" that need to be performed. It will n
 
 ## 4. Run `dnscontrol push`
 
-This will actually perform the required changes with the various providers.
+This will actually generate `zones/example.com.zone` for you. The bind provider is more configurable, and you can read more information [here.]({{site.github.url}}/providers/bind)

integrationTest/providers.json

@@ -9,5 +9,8 @@
         "domain": "$DNSIMPLE_DOMAIN",
         "token": "$DNSIMPLE_TOKEN",
         "baseurl": "https://api.sandbox.dnsimple.com"
+    },
+    "BIND":{
+        "domain": "example.com"
     }
 }

integrationTest/zones/example.com.zone

@@ -0,0 +1,4 @@
+$TTL 300
+@                IN SOA   DEFAULT_NOT_SET. DEFAULT_NOT_SET. 2017032059 3600 600 604800 1440
+                IN NS foo.com.
+xn--o-0gab       IN CNAME xn--ndaaa.xn--ndaaa.

main.go

@@ -23,15 +23,12 @@ import (
 
 //go:generate go run build/generate/generate.go
 
-// One of these config options must be set.
 var jsFile = flag.String("js", "dnsconfig.js", "Javascript file containing dns config")
-var stdin = flag.Bool("stdin", false, "Read domain config JSON from stdin")
-var jsonInput = flag.String("json", "", "Read domain config from specified JSON file.")
+var credsFile = flag.String("creds", "creds.json", "Provider credentials JSON file")
 
 var jsonOutputPre = flag.String("debugrawjson", "", "Write JSON intermediate to this file pre-normalization.")
 var jsonOutputPost = flag.String("debugjson", "", "During preview, write JSON intermediate to this file instead of stdout.")
 
-var configFile = flag.String("creds", "creds.json", "Provider credentials JSON file")
 var devMode = flag.Bool("dev", false, "Use helpers.js from disk instead of embedded")
 
 var flagProviders = flag.String("providers", "", "Providers to enable (comma seperated list); default is all-but-bind. Specify 'all' for all (including bind)")
@@ -49,11 +46,7 @@ func main() {
 	}
 
 	var dnsConfig *models.DNSConfig
-	if *stdin {
-		log.Fatal("Read from stdin not implemented yet.")
-	} else if *jsonInput != "" {
-		log.Fatal("Direct JSON read not implemented")
-	} else if *jsFile != "" {
+	if *jsFile != "" {
 		text, err := ioutil.ReadFile(*jsFile)
 		if err != nil {
 			log.Fatalf("Error reading %v: %v\n", *jsFile, err)
@@ -117,9 +110,17 @@ func main() {
 		return
 	}
 
-	providerConfigs, err := config.LoadProviderConfigs(*configFile)
+	providerConfigs, err := config.LoadProviderConfigs(*credsFile)
 	if err != nil {
-		log.Fatalf("error loading provider configurations: %s", err)
+		log.Fatalf(err.Error())
+	}
+	nonDefaultProviders := []string{}
+	for name, vals := range providerConfigs {
+		// add "_exclude_from_defaults":"true" to a domain to exclude it from being run unless
+		// -providers=all or -providers=name
+		if vals["_exclude_from_defaults"] == "true" {
+			nonDefaultProviders = append(nonDefaultProviders, name)
+		}
 	}
 	registrars, err := providers.CreateRegistrars(dnsConfig, providerConfigs)
 	if err != nil {
@@ -168,7 +169,7 @@ func main() {
 				if err != nil {
 					log.Fatal(err)
 				}
-				shouldrun := shouldRunProvider(prov, dc)
+				shouldrun := shouldRunProvider(prov, dc, nonDefaultProviders)
 				statusLbl := ""
 				if !shouldrun {
 					statusLbl = "(skipping)"
@@ -197,7 +198,7 @@ func main() {
 				fmt.Printf("%d correction%s\n", len(corrections), plural)
 				anyErrors = printOrRunCorrections(corrections, command) || anyErrors
 			}
-			if run := shouldRunProvider(domain.Registrar, domain); !run {
+			if run := shouldRunProvider(domain.Registrar, domain, nonDefaultProviders); !run {
 				continue
 			}
 			fmt.Printf("----- Registrar: %s\n", domain.Registrar)
@@ -272,23 +273,17 @@ func printOrRunCorrections(corrections []*models.Correction, command string) (an
 	return anyErrors
 }
 
-func shouldRunProvider(p string, dc *models.DomainConfig) bool {
+func shouldRunProvider(p string, dc *models.DomainConfig, nonDefaultProviders []string) bool {
 	if *flagProviders == "all" {
 		return true
 	}
 	if *flagProviders == "" {
-		return (p != "bind")
-		// NOTE(tlim): Hardcoding bind is a hacky way to make it off by default.
-		// As a result, bind only runs if you list it in -providers or use
-		// -providers=all.
-		// If you always want bind to run, call it something else in dnsconfig.js
-		// for example `NewDSP('bindyes', 'BIND',`.
-		// We don't want this hack, but we shouldn't need this in the future
-		// so it doesn't make sense to write a lot of code to make it work.
-		// In the future, the above `return p != "bind"` can become `return true`.
-		// Alternatively we might want to add a complex system that permits
-		// fancy whitelist/blacklisting of providers with defaults and so on.
-		// In that case, all of this hack will go away.
+		for _, pr := range nonDefaultProviders {
+			if pr == p {
+				return false
+			}
+		}
+		return true
 	}
 	for _, prov := range strings.Split(*flagProviders, ",") {
 		if prov == p {

providers/bind/bindProvider.go

@@ -11,10 +11,6 @@ bind -
 	if an update is actually needed. The old zonefile is also used
 	as the basis for generating the new SOA serial number.
 
-	If -bind_skeletin_src and -bind_skeletin_dst is defined, a
-	recursive file copy is performed from src to dst.  This is
-	useful for copying named.ca and other static files.
-
 */
 
 import (
@@ -50,20 +46,20 @@ func (s SoaInfo) String() string {
 }
 
 type Bind struct {
-	Default_NS  []string             `json:"default_ns"`
-	Default_Soa SoaInfo              `json:"default_soa"`
-	nameservers []*models.Nameserver `json:"-"`
+	DefaultNS   []string `json:"default_ns"`
+	DefaultSoa  SoaInfo  `json:"default_soa"`
+	nameservers []*models.Nameserver
 }
 
 var bindBaseDir = flag.String("bindtree", "zones", "BIND: Directory that stores BIND zonefiles.")
 
 //var bindSkeletin = flag.String("bind_skeletin", "skeletin/master/var/named/chroot/var/named/master", "")
 
-func rrToRecord(rr dns.RR, origin string, replace_serial uint32) (models.RecordConfig, uint32) {
+func rrToRecord(rr dns.RR, origin string, replaceSerial uint32) (models.RecordConfig, uint32) {
 	// Convert's dns.RR into our native data type (models.RecordConfig).
 	// Records are translated directly with no changes.
 	// If it is an SOA for the apex domain and
-	// replace_serial != 0, change the serial to replace_serial.
+	// replaceSerial != 0, change the serial to replaceSerial.
 	// WARNING(tlim): This assumes SOAs do not have serial=0.
 	// If one is found, we replace it with serial=1.
 	var old_serial, new_serial uint32
@@ -92,8 +88,8 @@ func rrToRecord(rr dns.RR, origin string, replace_serial uint32) (models.RecordC
 			old_serial = 1
 		}
 		new_serial = v.Serial
-		if rc.Name == "@" && replace_serial != 0 {
-			new_serial = replace_serial
+		if rc.Name == "@" && replaceSerial != 0 {
+			new_serial = replaceSerial
 		}
 		rc.Target = fmt.Sprintf("%v %v %v %v %v %v %v",
 			v.Ns, v.Mbox, new_serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
@@ -107,12 +103,12 @@ func rrToRecord(rr dns.RR, origin string, replace_serial uint32) (models.RecordC
 
 func makeDefaultSOA(info SoaInfo, origin string) *models.RecordConfig {
 	// Make a default SOA record in case one isn't found:
-	soa_rec := models.RecordConfig{
+	soaRec := models.RecordConfig{
 		Type: "SOA",
 		Name: "@",
 	}
-	soa_rec.NameFQDN = dnsutil.AddOrigin(soa_rec.Name, origin)
-
+	soaRec.NameFQDN = dnsutil.AddOrigin(soaRec.Name, origin)
+	//TODO(cpeterson): are these sane defaults?
 	if len(info.Ns) == 0 {
 		info.Ns = "DEFAULT_NOT_SET"
 	}
@@ -134,9 +130,9 @@ func makeDefaultSOA(info SoaInfo, origin string) *models.RecordConfig {
 	if info.Minttl == 0 {
 		info.Minttl = 1440
 	}
-	soa_rec.Target = info.String()
+	soaRec.Target = info.String()
 
-	return &soa_rec
+	return &soaRec
 }
 
 func (c *Bind) GetNameservers(string) ([]*models.Nameserver, error) {
@@ -144,7 +140,7 @@ func (c *Bind) GetNameservers(string) ([]*models.Nameserver, error) {
 }
 
 func (c *Bind) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
-
+	dc.Punycode()
 	// Phase 1: Copy everything to []*models.RecordConfig:
 	//    expectedRecords < dc.Records[i]
 	//    foundRecords < zonefile
@@ -159,34 +155,34 @@ func (c *Bind) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correcti
 	// diff.Inc...(foundDiffRecords, expectedDiffRecords )
 
 	// Default SOA record.  If we see one in the zone, this will be replaced.
-	soa_rec := makeDefaultSOA(c.Default_Soa, dc.Name)
+	soaRec := makeDefaultSOA(c.DefaultSoa, dc.Name)
 
 	// Read foundRecords:
 	foundRecords := make([]*models.RecordConfig, 0)
-	var old_serial, new_serial uint32
+	var oldSerial, newSerial uint32
 	zonefile := filepath.Join(*bindBaseDir, strings.ToLower(dc.Name)+".zone")
-	found_fh, err := os.Open(zonefile)
-	zone_file_found := err == nil
+	foundFH, err := os.Open(zonefile)
+	zoneFileFound := err == nil
 	if err != nil && !os.IsNotExist(os.ErrNotExist) {
 		// Don't whine if the file doesn't exist. However all other
 		// errors will be reported.
 		fmt.Printf("Could not read zonefile: %v\n", err)
 	} else {
-		for x := range dns.ParseZone(found_fh, dc.Name, zonefile) {
+		for x := range dns.ParseZone(foundFH, dc.Name, zonefile) {
 			if x.Error != nil {
 				log.Println("Error in zonefile:", x.Error)
 			} else {
-				rec, serial := rrToRecord(x.RR, dc.Name, old_serial)
-				if serial != 0 && old_serial != 0 {
+				rec, serial := rrToRecord(x.RR, dc.Name, oldSerial)
+				if serial != 0 && oldSerial != 0 {
 					log.Fatalf("Multiple SOA records in zonefile: %v\n", zonefile)
 				}
 				if serial != 0 {
 					// This was an SOA record. Update the serial.
-					old_serial = serial
-					new_serial = generate_serial(old_serial)
+					oldSerial = serial
+					newSerial = generate_serial(oldSerial)
 					// Regenerate with new serial:
-					*soa_rec, _ = rrToRecord(x.RR, dc.Name, new_serial)
-					rec = *soa_rec
+					*soaRec, _ = rrToRecord(x.RR, dc.Name, newSerial)
+					rec = *soaRec
 				}
 				foundRecords = append(foundRecords, &rec)
 			}
@@ -195,7 +191,7 @@ func (c *Bind) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correcti
 
 	// Add SOA record to expected set:
 	if !dc.HasRecordTypeName("SOA", "@") {
-		dc.Records = append(dc.Records, soa_rec)
+		dc.Records = append(dc.Records, soaRec)
 	}
 
 	differ := diff.New(dc)
@@ -206,24 +202,24 @@ func (c *Bind) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correcti
 	changes := false
 	for _, i := range create {
 		changes = true
-		if zone_file_found {
+		if zoneFileFound {
 			fmt.Fprintln(buf, i)
 		}
 	}
 	for _, i := range del {
 		changes = true
-		if zone_file_found {
+		if zoneFileFound {
 			fmt.Fprintln(buf, i)
 		}
 	}
 	for _, i := range mod {
 		changes = true
-		if zone_file_found {
+		if zoneFileFound {
 			fmt.Fprintln(buf, i)
 		}
 	}
 	msg := fmt.Sprintf("GENERATE_ZONEFILE: %s\n", dc.Name)
-	if !zone_file_found {
+	if !zoneFileFound {
 		msg = msg + fmt.Sprintf(" (%d records)\n", len(create))
 	}
 	msg += buf.String()
@@ -270,7 +266,7 @@ func initBind(config map[string]string, providermeta json.RawMessage) (providers
 			return nil, err
 		}
 	}
-	api.nameservers = models.StringsToNameservers(api.Default_NS)
+	api.nameservers = models.StringsToNameservers(api.DefaultNS)
 	return api, nil
 }
 

providers/config/providerConfig.go

@@ -19,6 +19,10 @@ func LoadProviderConfigs(fname string) (map[string]map[string]string, error) {
 	var results = map[string]map[string]string{}
 	dat, err := utfutil.ReadFile(fname, utfutil.POSIX)
 	if err != nil {
+		//no creds file is ok. Bind requires nothing for example. Individual providers will error if things not found.
+		if os.IsNotExist(err) {
+			return results, nil
+		}
 		return nil, fmt.Errorf("While reading provider credentials file %v: %v", fname, err)
 	}
 	s := string(dat)

providers/providers.go

@@ -88,12 +88,8 @@ func CreateRegistrars(d *models.DNSConfig, providerConfigs map[string]map[string
 func CreateDsps(d *models.DNSConfig, providerConfigs map[string]map[string]string) (map[string]DNSServiceProvider, error) {
 	dsps := map[string]DNSServiceProvider{}
 	for _, dsp := range d.DNSProviders {
-		//log.Printf("dsp.Name=%#v\n", dsp.Name)
-		rawMsg, ok := providerConfigs[dsp.Name]
-		if !ok {
-			return nil, fmt.Errorf("DNSServiceProvider %s not listed in -providers file", dsp.Name)
-		}
-		provider, err := CreateDNSProvider(dsp.Type, rawMsg, dsp.Metadata)
+		vals := providerConfigs[dsp.Name]
+		provider, err := CreateDNSProvider(dsp.Type, vals, dsp.Metadata)
 		if err != nil {
 			return nil, err
 		}