NEW PROVIDER: deSEC (#725)

* Add initial deSEC support

* Handle the api rate limiting

* Fix deleteRR and do some code cleanup

* improve rate limiting and record deletion

* Add documentation for deSEC provider

* README.md update list of supported DNS providers

* deSEC supports SSHFP records

* dynamic minimum_ttl and hint for DNSSec on domain creation

* merge all changes into one single bulk api request

* Fix: actually set the TTL to min_ttl if necessary

* use a constant for apiBase URL

* Fix code comments

* Use PUT instead of PATCH for upsertRR method

* use ' instead of " for java script examples

README.md

@@ -20,6 +20,7 @@ Currently supported DNS providers:
  - BIND
  - ClouDNS
  - Cloudflare
+ - deSEC
  - DNSimple
  - DigitalOcean
  - Exoscale

docs/_providers/desec.md

@@ -0,0 +1,36 @@
+---
+name: deSEC
+title: deSEC Provider
+layout: default
+jsId: DESEC
+---
+# deSEC Provider
+## Configuration
+In your providers credentials file you must provide a deSEC account auth token:
+
+{% highlight json %}
+{
+  "desec": {
+    "auth-token": "your-deSEC-auth-token"
+  }
+}
+{% endhighlight %}
+
+## Metadata
+This provider does not recognize any special metadata fields unique to deSEC.
+
+## Usage
+Example Javascript:
+
+{% highlight js %}
+var REG_NONE = NewRegistrar('none', 'NONE');    // No registrar.
+var deSEC = NewDnsProvider('desec', 'DESEC');  // deSEC
+
+D('example.tld', REG_NONE, DnsProvider(deSEC),
+    A('test','1.2.3.4')
+);
+{% endhighlight %}
+
+## Activation
+DNSControl depends on a deSEC account auth token.
+This token can be obtained by logging in via the deSEC API: https://desec.readthedocs.io/en/latest/auth/account.html#log-in

providers/_all/all.go

@@ -8,6 +8,7 @@ import (
 	_ "github.com/StackExchange/dnscontrol/v3/providers/bind"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/cloudflare"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/cloudns"
+	_ "github.com/StackExchange/dnscontrol/v3/providers/desec"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/digitalocean"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/dnsimple"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/exoscale"

providers/desec/convert.go

@@ -0,0 +1,79 @@
+package desec
+
+// Convert the provider's native record description to models.RecordConfig.
+
+import (
+	"fmt"
+
+	"github.com/StackExchange/dnscontrol/v3/models"
+	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
+)
+
+// nativeToRecord takes a DNS record from deSEC and returns a native RecordConfig struct.
+func nativeToRecords(n resourceRecord, origin string) (rcs []*models.RecordConfig) {
+
+	// deSEC returns all the values for a given label/rtype pair in each
+	// resourceRecord.  In other words, if there are multiple A
+	// records for a label, all the IP addresses are listed in
+	// n.Records rather than having many resourceRecord's.
+	// We must split them out into individual records, one for each value.
+	for _, value := range n.Records {
+		rc := &models.RecordConfig{
+			TTL:      n.TTL,
+			Original: n,
+		}
+		rc.SetLabel(n.Subname, origin)
+		switch rtype := n.Type; rtype {
+		default: //  "A", "AAAA", "CAA", "NS", "CNAME", "MX", "PTR", "SRV", "TXT"
+			if err := rc.PopulateFromString(rtype, value, origin); err != nil {
+				panic(fmt.Errorf("unparsable record received from deSEC: %w", err))
+			}
+		}
+		rcs = append(rcs, rc)
+	}
+
+	return rcs
+}
+
+func recordsToNative(rcs []*models.RecordConfig, origin string) []resourceRecord {
+	// Take a list of RecordConfig and return an equivalent list of resourceRecord.
+	// deSEC requires one resourceRecord for each label:key tuple, therefore we
+	// might collapse many RecordConfig into one resourceRecord.
+
+	var keys = map[models.RecordKey]*resourceRecord{}
+	var zrs []resourceRecord
+
+	for _, r := range rcs {
+		label := r.GetLabel()
+		if label == "@" {
+			label = ""
+		}
+		key := r.Key()
+
+		if zr, ok := keys[key]; !ok {
+			// Allocate a new ZoneRecord:
+			zr := resourceRecord{
+				Type:    r.Type,
+				TTL:     r.TTL,
+				Subname: label,
+				Records: []string{r.GetTargetCombined()},
+			}
+			zrs = append(zrs, zr)
+			//keys[key] = &zr   // This didn't work.
+			keys[key] = &zrs[len(zrs)-1] // This does work. I don't know why.
+
+		} else {
+			zr.Records = append(zr.Records, r.GetTargetCombined())
+
+			if r.TTL != zr.TTL {
+				printer.Warnf("All TTLs for a rrset (%v) must be the same. Using smaller of %v and %v.\n", key, r.TTL, zr.TTL)
+				if r.TTL < zr.TTL {
+					zr.TTL = r.TTL
+				}
+			}
+
+		}
+	}
+
+	return zrs
+}

providers/desec/desecProvider.go

@@ -0,0 +1,218 @@
+package desec
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/StackExchange/dnscontrol/v3/models"
+	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
+	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
+	"github.com/StackExchange/dnscontrol/v3/providers"
+	"github.com/miekg/dns/dnsutil"
+)
+
+/*
+desec API DNS provider:
+Info required in `creds.json`:
+   - auth-token
+*/
+
+// NewDeSec creates the provider.
+func NewDeSec(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
+	c := &api{}
+	c.creds.token = m["auth-token"]
+	if c.creds.token == "" {
+		return nil, fmt.Errorf("missing deSEC auth-token")
+	}
+
+	// Get a domain to validate authentication
+	if err := c.fetchDomainList(); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+var features = providers.DocumentationNotes{
+	providers.DocDualHost:            providers.Unimplemented(),
+	providers.DocOfficiallySupported: providers.Cannot(),
+	providers.DocCreateDomains:       providers.Can(),
+	providers.CanUseAlias:            providers.Cannot(),
+	providers.CanUseSRV:              providers.Can(),
+	providers.CanUseSSHFP:            providers.Can(),
+	providers.CanUseCAA:              providers.Can(),
+	providers.CanUseTLSA:             providers.Can(),
+	providers.CanUsePTR:              providers.Unimplemented(),
+	providers.CanGetZones:            providers.Can(),
+	providers.CanAutoDNSSEC:          providers.Cannot(),
+}
+
+var defaultNameServerNames = []string{
+	"ns1.desec.io",
+	"ns2.desec.org",
+}
+
+func init() {
+	providers.RegisterDomainServiceProviderType("DESEC", NewDeSec, features)
+}
+
+// GetNameservers returns the nameservers for a domain.
+func (c *api) GetNameservers(domain string) ([]*models.Nameserver, error) {
+	return models.ToNameservers(defaultNameServerNames)
+}
+
+func (c *api) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
+	existing, err := c.GetZoneRecords(dc.Name)
+	if err != nil {
+		return nil, err
+	}
+	models.PostProcessRecords(existing)
+	clean := PrepFoundRecords(existing)
+	var min_ttl uint32
+	if ttl, ok := c.domainIndex[dc.Name]; !ok {
+		min_ttl = 3600
+	} else {
+		min_ttl = ttl
+	}
+	PrepDesiredRecords(dc, min_ttl)
+	return c.GenerateDomainCorrections(dc, clean)
+}
+
+// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
+func (c *api) GetZoneRecords(domain string) (models.Records, error) {
+	records, err := c.getRecords(domain)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert them to DNScontrol's native format:
+	existingRecords := []*models.RecordConfig{}
+	for _, rr := range records {
+		existingRecords = append(existingRecords, nativeToRecords(rr, domain)...)
+	}
+	return existingRecords, nil
+}
+
+// EnsureDomainExists returns an error if domain doesn't exist.
+func (c *api) EnsureDomainExists(domain string) error {
+	if err := c.fetchDomainList(); err != nil {
+		return err
+	}
+	// domain already exists
+	if _, ok := c.domainIndex[domain]; ok {
+		return nil
+	}
+	return c.createDomain(domain)
+}
+
+// PrepFoundRecords munges any records to make them compatible with
+// this provider. Usually this is a no-op.
+func PrepFoundRecords(recs models.Records) models.Records {
+	// If there are records that need to be modified, removed, etc. we
+	// do it here.  Usually this is a no-op.
+	return recs
+}
+
+// PrepDesiredRecords munges any records to best suit this provider.
+func PrepDesiredRecords(dc *models.DomainConfig, min_ttl uint32) {
+	// Sort through the dc.Records, eliminate any that can't be
+	// supported; modify any that need adjustments to work with the
+	// provider.  We try to do minimal changes otherwise it gets
+	// confusing.
+
+	dc.Punycode()
+	recordsToKeep := make([]*models.RecordConfig, 0, len(dc.Records))
+	for _, rec := range dc.Records {
+		if rec.Type == "ALIAS" {
+			// deSEC does not permit ALIAS records, just ignore it
+			printer.Warnf("deSEC does not support alias records\n")
+			continue
+		}
+		if rec.TTL < min_ttl {
+			if rec.Type != "NS" {
+				printer.Warnf("Please contact support@desec.io if you need ttls < %d. Setting ttl of %s type %s from %d to %d\n", min_ttl, rec.GetLabelFQDN(), rec.Type, rec.TTL, min_ttl)
+			}
+			rec.TTL = min_ttl
+		}
+		recordsToKeep = append(recordsToKeep, rec)
+	}
+	dc.Records = recordsToKeep
+}
+
+// GenerateDomainCorrections takes the desired and existing records
+// and produces a Correction list.  The correction list is simply
+// a list of functions to call to actually make the desired
+// correction, and a message to output to the user when the change is
+// made.
+func (client *api) GenerateDomainCorrections(dc *models.DomainConfig, existing models.Records) ([]*models.Correction, error) {
+
+	var corrections = []*models.Correction{}
+
+	// diff existing vs. current.
+	differ := diff.New(dc)
+	keysToUpdate := differ.ChangedGroups(existing)
+	if len(keysToUpdate) == 0 {
+		return nil, nil
+	}
+
+	desiredRecords := dc.Records.GroupedByKey()
+	var rrs []resourceRecord
+	buf := &bytes.Buffer{}
+	// For any key with an update, delete or replace those records.
+	for label := range keysToUpdate {
+		if _, ok := desiredRecords[label]; !ok {
+			//we could not find this RecordKey in the desiredRecords
+			//this means it must be deleted
+			for i, msg := range keysToUpdate[label] {
+				if i == 0 {
+					rc := resourceRecord{}
+					rc.Type = label.Type
+					rc.Records = make([]string, 0) // empty array of records should delete this rrset
+					rc.TTL = 3600
+					shortname := dnsutil.TrimDomainName(label.NameFQDN, dc.Name)
+					if shortname == "@" {
+						shortname = ""
+					}
+					rc.Subname = shortname
+					fmt.Fprintln(buf, msg)
+					rrs = append(rrs, rc)
+				} else {
+					//just add the message
+					fmt.Fprintln(buf, msg)
+				}
+			}
+		} else {
+			//it must be an update or create, both can be done with the same api call.
+			ns := recordsToNative(desiredRecords[label], dc.Name)
+			if len(ns) > 1 {
+				panic("we got more than one resource record to create / modify")
+			}
+			for i, msg := range keysToUpdate[label] {
+				if i == 0 {
+					rrs = append(rrs, ns[0])
+					fmt.Fprintln(buf, msg)
+				} else {
+					//noop just for printing the additional messages
+					fmt.Fprintln(buf, msg)
+				}
+			}
+		}
+	}
+	var msg string
+	msg = fmt.Sprintf("Changes:\n%s", buf)
+	corrections = append(corrections,
+		&models.Correction{
+			Msg: msg,
+			F: func() error {
+				rc := rrs
+				err := client.upsertRR(rc, dc.Name)
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+		})
+
+	return corrections, nil
+}

providers/desec/protocol.go

@@ -0,0 +1,217 @@
+package desec
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
+)
+
+const apiBase = "https://desec.io/api/v1"
+
+// Api layer for desec
+type api struct {
+	domainIndex      map[string]uint32
+	nameserversNames []string
+	creds            struct {
+		tokenid  string
+		token    string
+		user     string
+		password string
+	}
+}
+
+type domainObject struct {
+	Created    time.Time   `json:"created,omitempty"`
+	Keys       []dnssecKey `json:"keys,omitempty"`
+	MinimumTTL uint32      `json:"minimum_ttl,omitempty"`
+	Name       string      `json:"name,omitempty"`
+	Published  time.Time   `json:"published,omitempty"`
+}
+
+type resourceRecord struct {
+	Subname string   `json:"subname"`
+	Records []string `json:"records"`
+	TTL     uint32   `json:"ttl,omitempty"`
+	Type    string   `json:"type"`
+	Target  string   `json:"-"`
+}
+
+type rrResponse struct {
+	resourceRecord
+	Created time.Time `json:"created"`
+	Domain  string    `json:"domain"`
+	Name    string    `json:"name"`
+}
+
+type dnssecKey struct {
+	Dnskey  string   `json:"dnskey"`
+	Ds      []string `json:"ds"`
+	Flags   int      `json:"flags"`
+	Keytype string   `json:"keytype"`
+}
+
+type errorResponse struct {
+	Detail string `json:"detail"`
+}
+
+func (c *api) fetchDomainList() error {
+	c.domainIndex = map[string]uint32{}
+	var dr []domainObject
+	endpoint := "/domains/"
+	var bodyString, err = c.get(endpoint, "GET")
+	if err != nil {
+		return fmt.Errorf("Error fetching domain list from deSEC: %s", err)
+	}
+	err = json.Unmarshal(bodyString, &dr)
+	if err != nil {
+		return err
+	}
+	for _, domain := range dr {
+		//We store the min ttl in the domain index
+		//This will be used for validation and auto correction
+		c.domainIndex[domain.Name] = domain.MinimumTTL
+	}
+	return nil
+}
+
+func (c *api) getRecords(domain string) ([]resourceRecord, error) {
+	endpoint := "/domains/%s/rrsets/"
+	var rrs []rrResponse
+	var rrs_new []resourceRecord
+	var bodyString, err = c.get(fmt.Sprintf(endpoint, domain), "GET")
+	if err != nil {
+		return rrs_new, fmt.Errorf("Error fetching records from deSEC for domain %s: %s", domain, err)
+	}
+	err = json.Unmarshal(bodyString, &rrs)
+	if err != nil {
+		return rrs_new, err
+	}
+	// deSEC returns round robin records as array but dnsconfig expects single entries for each record
+	// we will create one object per record except of TXT records which are handled as array of string by dnscontrol aswell.
+	for i := range rrs {
+		tmp := resourceRecord{
+			TTL:     rrs[i].TTL,
+			Type:    rrs[i].Type,
+			Subname: rrs[i].Subname,
+			Records: rrs[i].Records,
+		}
+		rrs_new = append(rrs_new, tmp)
+	}
+	return rrs_new, nil
+}
+
+func (c *api) createDomain(domain string) error {
+	endpoint := "/domains/"
+	pl := domainObject{Name: domain}
+	byt, _ := json.Marshal(pl)
+	var resp []byte
+	var err error
+	if resp, err = c.post(endpoint, "POST", byt); err != nil {
+		return fmt.Errorf("Error create domain deSEC: %v", err)
+	}
+	dm := domainObject{}
+	err = json.Unmarshal(resp, &dm)
+	if err != nil {
+		return err
+	}
+	printer.Printf("If you want to use DNSSec please add the DS record at your registrar using one of the keys:\n")
+	printer.Printf("%+q", dm.Keys)
+	return nil
+}
+
+//upsertRR will create or override the RRSet with the provided resource record.
+func (c *api) upsertRR(rr []resourceRecord, domain string) error {
+	endpoint := fmt.Sprintf("/domains/%s/rrsets/", domain)
+	byt, _ := json.Marshal(rr)
+	if _, err := c.post(endpoint, "PUT", byt); err != nil {
+		return fmt.Errorf("Error create rrset deSEC: %v", err)
+	}
+	return nil
+}
+
+func (c *api) deleteRR(domain, shortname, t string) error {
+	endpoint := fmt.Sprintf("/domains/%s/rrsets/%s/%s/", domain, shortname, t)
+	if _, err := c.get(endpoint, "DELETE"); err != nil {
+		return fmt.Errorf("Error delete rrset deSEC: %v", err)
+	}
+	return nil
+}
+
+func (c *api) get(endpoint, method string) ([]byte, error) {
+	retrycnt := 0
+retry:
+	client := &http.Client{}
+	req, _ := http.NewRequest(method, apiBase+endpoint, nil)
+	q := req.URL.Query()
+	req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))
+
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	bodyString, _ := ioutil.ReadAll(resp.Body)
+	// Got error from API ?
+	if resp.StatusCode > 299 {
+		if resp.StatusCode == 429 && retrycnt < 5 {
+			retrycnt++
+			time.Sleep(500 * time.Millisecond)
+			goto retry
+		}
+		var errResp errorResponse
+		err = json.Unmarshal(bodyString, &errResp)
+		if err == nil {
+			return bodyString, fmt.Errorf("%s", errResp.Detail)
+		}
+		return bodyString, fmt.Errorf("http status %d %s, the api does not provide more information", resp.StatusCode, resp.Status)
+	}
+	return bodyString, nil
+}
+
+func (c *api) post(endpoint, method string, payload []byte) ([]byte, error) {
+	retrycnt := 0
+retry:
+	client := &http.Client{}
+	req, err := http.NewRequest(method, apiBase+endpoint, bytes.NewReader(payload))
+	if err != nil {
+		return []byte{}, err
+	}
+	q := req.URL.Query()
+	if endpoint != "/auth/login/" {
+		req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	bodyString, _ := ioutil.ReadAll(resp.Body)
+
+	// Got error from API ?
+	if resp.StatusCode > 299 {
+		if resp.StatusCode == 429 && retrycnt < 5 {
+			retrycnt++
+			time.Sleep(500 * time.Millisecond)
+			goto retry
+		}
+		var errResp errorResponse
+		err = json.Unmarshal(bodyString, &errResp)
+		if err == nil {
+			return bodyString, fmt.Errorf("http status %d %s details: %s", resp.StatusCode, resp.Status, errResp.Detail)
+		}
+		return bodyString, fmt.Errorf("http status %d %s, the api does not provide more information", resp.StatusCode, resp.Status)
+	}
+	//time.Sleep(334 * time.Millisecond)
+	return bodyString, nil
+}