From 01a242480c7b39db1926abb67a2a9847c3d2195d Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Thu, 25 May 2017 12:27:36 -0400 Subject: [PATCH] Initial DNS Resolvers and SPF scaffolding (#123) * Implemented Live and Preloaded resolvers * Integrated Craig's parser. --- cmd/spftest/main.go | 64 +++++++++++++++++++++++ dnsresolver/dnscache.go | 28 +++++++++++ dnsresolver/dnscache_test.go | 31 ++++++++++++ dnsresolver/resolver.go | 83 ++++++++++++++++++++++++++++++ spflib/parse.go | 98 ++++++++++++++++++++++++++++++++++++ spflib/parse_test.go | 30 +++++++++++ spflib/testdata-dns1.json | 64 +++++++++++++++++++++++ 7 files changed, 398 insertions(+) create mode 100644 cmd/spftest/main.go create mode 100644 dnsresolver/dnscache.go create mode 100644 dnsresolver/dnscache_test.go create mode 100644 dnsresolver/resolver.go create mode 100644 spflib/parse.go create mode 100644 spflib/parse_test.go create mode 100644 spflib/testdata-dns1.json diff --git a/cmd/spftest/main.go b/cmd/spftest/main.go new file mode 100644 index 000000000..df3a86c96 --- /dev/null +++ b/cmd/spftest/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/StackExchange/dnscontrol/dnsresolver" + "github.com/StackExchange/dnscontrol/spflib" +) + +func main() { + + h := dnsresolver.NewResolverLive("spf-store.json") + fmt.Println(h.GetTxt("_spf.google.com")) + fmt.Println(h.GetTxt("spf-basic.fogcreek.com")) + h.Close() + + i, err := dnsresolver.NewResolverPreloaded("spf-store.json") + if err != nil { + panic(err) + } + fmt.Println(i.GetTxt("_spf.google.com")) + fmt.Println(i.GetTxt("spf-basic.fogcreek.com")) + fmt.Println(i.GetTxt("wontbefound")) + + fmt.Println() + fmt.Println("---------------------") + fmt.Println() + + res := dnsresolver.NewResolverLive("preload-dns.json") + //res := dnsresolver.NewResolverPreloaded("preload-dns.json") + + rec, err := spflib.Parse(strings.Join([]string{"v=spf1", + "ip4:198.252.206.0/24", + "ip4:192.111.0.0/24", + "include:_spf.google.com", + "include:mailgun.org", + "include:spf-basic.fogcreek.com", + "include:mail.zendesk.com", + "include:servers.mcsv.net", + "include:sendgrid.net", + "include:spf.mtasv.net", + "~all"}, " "), res) + if err != nil { + panic(err) + } + spflib.DumpSPF(rec, "") + + fmt.Println() + fmt.Println("---------------------") + fmt.Println() + + var spfs []string + spfs, err = spflib.Lookup("stackex.com", res) + if err != nil { + panic(err) + } + rec, err = spflib.Parse(strings.Join(spfs, " "), res) + if err != nil { + panic(err) + } + spflib.DumpSPF(rec, "") + +} diff --git a/dnsresolver/dnscache.go b/dnsresolver/dnscache.go new file mode 100644 index 000000000..edb102d04 --- /dev/null +++ b/dnsresolver/dnscache.go @@ -0,0 +1,28 @@ +package dnsresolver + +// dnsCache implements a very simple DNS cache. +// It caches the entire answer (i.e. all TXT records), filtering +// out the non-SPF answers is done at a higher layer. +// At this time the only rtype is "TXT". Eventually we'll need +// to cache A/AAAA/CNAME records to to CNAME flattening. +type dnsCache map[string]map[string][]string // map[fqdn]map[rtype] -> answers + +func (c dnsCache) get(label, rtype string) ([]string, bool) { + v1, ok := c[label] + if !ok { + return nil, false + } + v2, ok := v1[rtype] + if !ok { + return nil, false + } + return v2, true +} + +func (c dnsCache) put(label, rtype string, answers []string) { + _, ok := c[label] + if !ok { + c[label] = make(map[string][]string) + } + c[label][rtype] = answers +} diff --git a/dnsresolver/dnscache_test.go b/dnsresolver/dnscache_test.go new file mode 100644 index 000000000..e4f32845c --- /dev/null +++ b/dnsresolver/dnscache_test.go @@ -0,0 +1,31 @@ +package dnsresolver + +import "testing" + +func TestDnsCache(t *testing.T) { + + cache := &dnsCache{} + cache.put("one", "txt", []string{"a", "b", "c"}) + cache.put("two", "txt", []string{"d", "e", "f"}) + + a, b := cache.get("one", "txt") + if !(b == true && len(a) == 3 && a[0] == "a" && a[1] == "b" && a[2] == "c") { + t.Errorf("one-txt didn't work") + } + + a, b = cache.get("two", "txt") + if !(b == true && len(a) == 3 && a[0] == "d" && a[1] == "e" && a[2] == "f") { + t.Errorf("one-txt didn't work") + } + + a, b = cache.get("three", "txt") + if !(b == false) { + t.Errorf("three-txt didn't work") + } + + a, b = cache.get("two", "not") + if !(b == false) { + t.Errorf("two-not didn't work") + } + +} diff --git a/dnsresolver/resolver.go b/dnsresolver/resolver.go new file mode 100644 index 000000000..706dda89d --- /dev/null +++ b/dnsresolver/resolver.go @@ -0,0 +1,83 @@ +package dnsresolver + +import ( + "encoding/json" + "io/ioutil" + "net" + + "github.com/pkg/errors" +) + +// This file includes all the DNS Resolvers used by package spf. + +// DnsResolver looks up txt strings associated with a FQDN. +type DnsResolver interface { + GetTxt(string) ([]string, error) // Given a DNS label, return the TXT values records. +} + +// The "Live DNS" Resolver: + +type dnsLive struct { + filename string + cache dnsCache +} + +func NewResolverLive(filename string) *dnsLive { + // Does live DNS lookups. Records them. Writes file on Close. + c := &dnsLive{filename: filename} + c.cache = dnsCache{} + return c +} + +func (c *dnsLive) GetTxt(label string) ([]string, error) { + // Try the cache. + txts, ok := c.cache.get(label, "txt") + if ok { + return txts, nil + } + + // Populate the cache: + t, err := net.LookupTXT(label) + if err == nil { + c.cache.put(label, "txt", t) + } + + return t, err +} + +func (c *dnsLive) Close() { + // Write out and close the file. + m, _ := json.MarshalIndent(c.cache, "", " ") + m = append(m, "\n"...) + ioutil.WriteFile(c.filename, m, 0666) +} + +// The "Pre-Cached DNS" Resolver: + +type dnsPreloaded struct { + cache dnsCache +} + +func NewResolverPreloaded(filename string) (*dnsPreloaded, error) { + c := &dnsPreloaded{} + c.cache = dnsCache{} + j, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + err = json.Unmarshal(j, &(*c).cache) + return c, err +} + +func (c *dnsPreloaded) DumpCache() dnsCache { + return c.cache +} + +func (c *dnsPreloaded) GetTxt(label string) ([]string, error) { + // Try the cache. + txts, ok := c.cache.get(label, "txt") + if ok { + return txts, nil + } + return nil, errors.Errorf("No preloaded DNS entry for: %#v", label) +} diff --git a/spflib/parse.go b/spflib/parse.go new file mode 100644 index 000000000..37cb61a4c --- /dev/null +++ b/spflib/parse.go @@ -0,0 +1,98 @@ +package spflib + +import ( + "fmt" + "strings" + + "github.com/StackExchange/dnscontrol/dnsresolver" +) + +type SPFRecord struct { + Lookups int + Parts []*SPFPart +} + +type SPFPart struct { + Text string + Lookups int + IncludeRecord *SPFRecord +} + +func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) { + txts, err := dnsres.GetTxt(target) + if err != nil { + return nil, err + } + var result []string + for _, txt := range txts { + if strings.HasPrefix(txt, "v=spf1 ") { + result = append(result, txt) + } + } + return result, nil +} + +func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) { + if !strings.HasPrefix(text, "v=spf1 ") { + return nil, fmt.Errorf("Not an spf record") + } + parts := strings.Split(text, " ") + rec := &SPFRecord{} + for _, part := range parts[1:] { + p := &SPFPart{Text: part} + rec.Parts = append(rec.Parts, p) + if part == "~all" || part == "-all" || part == "?all" { + //all. nothing else matters. + break + } else if strings.HasPrefix(part, "ip4:") || strings.HasPrefix(part, "ip6:") { + //ip address, 0 lookups + continue + } else if strings.HasPrefix(part, "include:") { + rec.Lookups++ + includeTarget := strings.TrimPrefix(part, "include:") + subRecord, err := resolveSPF(includeTarget, dnsres) + if err != nil { + return nil, err + } + p.IncludeRecord, err = Parse(subRecord, dnsres) + if err != nil { + return nil, fmt.Errorf("In included spf: %s", err) + } + rec.Lookups += p.IncludeRecord.Lookups + } else { + return nil, fmt.Errorf("Unsupported spf part %s", part) + } + + } + return rec, nil +} + +// DumpSPF outputs an SPFRecord and related data for debugging purposes. +func DumpSPF(rec *SPFRecord, indent string) { + fmt.Printf("%sTotal Lookups: %d\n", indent, rec.Lookups) + fmt.Print(indent + "v=spf1") + for _, p := range rec.Parts { + fmt.Print(" " + p.Text) + } + fmt.Println() + indent += "\t" + for _, p := range rec.Parts { + if p.IncludeRecord != nil { + fmt.Println(indent + p.Text) + DumpSPF(p.IncludeRecord, indent+"\t") + } + } +} + +func resolveSPF(target string, dnsres dnsresolver.DnsResolver) (string, error) { + recs, err := dnsres.GetTxt(target) + if err != nil { + return "", err + } + for _, r := range recs { + if strings.HasPrefix(r, "v=spf1 ") { + return r, nil + } + } + return "", fmt.Errorf("No SPF records found for %s", target) +} diff --git a/spflib/parse_test.go b/spflib/parse_test.go new file mode 100644 index 000000000..aee030509 --- /dev/null +++ b/spflib/parse_test.go @@ -0,0 +1,30 @@ +package spflib + +import ( + "strings" + "testing" + + "github.com/StackExchange/dnscontrol/dnsresolver" +) + +func TestParse(t *testing.T) { + dnsres, err := dnsresolver.NewResolverPreloaded("testdata-dns1.json") + if err != nil { + t.Fatal(err) + } + rec, err := Parse(strings.Join([]string{"v=spf1", + "ip4:198.252.206.0/24", + "ip4:192.111.0.0/24", + "include:_spf.google.com", + "include:mailgun.org", + "include:spf-basic.fogcreek.com", + "include:mail.zendesk.com", + "include:servers.mcsv.net", + "include:sendgrid.net", + "include:spf.mtasv.net", + "~all"}, " "), dnsres) + if err != nil { + t.Fatal(err) + } + DumpSPF(rec, "") +} diff --git a/spflib/testdata-dns1.json b/spflib/testdata-dns1.json new file mode 100644 index 000000000..15f8266fe --- /dev/null +++ b/spflib/testdata-dns1.json @@ -0,0 +1,64 @@ +{ + "_netblocks.google.com": { + "txt": [ + "v=spf1 ip4:64.18.0.0/20 ip4:64.233.160.0/19 ip4:66.102.0.0/20 ip4:66.249.80.0/20 ip4:72.14.192.0/18 ip4:74.125.0.0/16 ip4:108.177.8.0/21 ip4:173.194.0.0/16 ip4:207.126.144.0/20 ip4:209.85.128.0/17 ip4:216.58.192.0/19 ip4:216.239.32.0/19 ~all" + ] + }, + "_netblocks2.google.com": { + "txt": [ + "v=spf1 ip6:2001:4860:4000::/36 ip6:2404:6800:4000::/36 ip6:2607:f8b0:4000::/36 ip6:2800:3f0:4000::/36 ip6:2a00:1450:4000::/36 ip6:2c0f:fb50:4000::/36 ~all" + ] + }, + "_netblocks3.google.com": { + "txt": [ + "v=spf1 ip4:172.217.0.0/19 ip4:108.177.96.0/19 ~all" + ] + }, + "_spf.google.com": { + "txt": [ + "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all" + ] + }, + "mail.zendesk.com": { + "txt": [ + "v=spf1 ip4:192.161.144.0/20 ip4:185.12.80.0/22 ip4:96.46.150.192/27 ip4:174.137.46.0/24 ip4:188.172.128.0/20 ip4:216.198.0.0/18 ~all" + ] + }, + "mailgun.org": { + "txt": [ + "google-site-verification=FIGVOKZm6lQFDBJaiC2DdwvBy8TInunoGCt-1gnL4PA", + "v=spf1 include:spf1.mailgun.org include:spf2.mailgun.org ~all" + ] + }, + "sendgrid.net": { + "txt": [ + "google-site-verification=NxyooVvVaIgddVa23KTlOEuVPuhffcDqJFV8RzWrAys", + "v=spf1 ip4:167.89.0.0/17 ip4:208.117.48.0/20 ip4:50.31.32.0/19 ip4:198.37.144.0/20 ip4:198.21.0.0/21 ip4:192.254.112.0/20 ip4:168.245.0.0/17 ~all" + ] + }, + "servers.mcsv.net": { + "txt": [ + "v=spf1 ip4:205.201.128.0/20 ip4:198.2.128.0/18 ip4:148.105.8.0/21 ?all" + ] + }, + "spf-basic.fogcreek.com": { + "txt": [ + "v=spf1 ip4:64.34.80.172 -all" + ] + }, + "spf.mtasv.net": { + "txt": [ + "v=spf1 ip4:50.31.156.96/27 ip4:104.245.209.192/26 ~all" + ] + }, + "spf1.mailgun.org": { + "txt": [ + "v=spf1 ip4:173.193.210.32/27 ip4:50.23.218.192/27 ip4:174.37.226.64/27 ip4:208.43.239.136/30 ip4:184.173.105.0/24 ip4:184.173.153.0/24 ip4:104.130.122.0/23 ip4:146.20.112.0/26 ~all" + ] + }, + "spf2.mailgun.org": { + "txt": [ + "v=spf1 ip4:209.61.151.0/24 ip4:166.78.68.0/22 ip4:198.61.254.0/23 ip4:192.237.158.0/23 ip4:23.253.182.0/23 ip4:104.130.96.0/28 ip4:146.20.113.0/24 ip4:146.20.191.0/24 ~all" + ] + } +}