NET-1603: Manage DNS NM changes (#3124)

* add switch for manage dns

* manage DNS sync publish

* add dns sync api

* add manageDNS field in peerUpdate

* add default dns for extClent if manage dns enabled

* add DEFAULT_DOMAIN for internal DNS lookup

* move DNSSync to peerUpdate

* fix empty host in network issue

* sync up dns when custom dns add/delete

* fix custom DNS ip4/ipv6 validator issue

config/config.go

@@ -100,6 +100,8 @@ type ServerConfig struct {
 	SmtpHost                   string        `json:"smtp_host"`
 	SmtpPort                   int           `json:"smtp_port"`
 	MetricInterval             string        `yaml:"metric_interval"`
+	ManageDNS                  bool          `yaml:"manage_dns"`
+	DefaultDomain              string        `yaml:"default_domain"`
 }
 
 // SQLConfig - Generic SQL Config

controllers/dns.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
 )
 
@@ -24,6 +25,8 @@ func dnsHandlers(r *mux.Router) {
 		Methods(http.MethodGet)
 	r.HandleFunc("/api/dns/adm/{network}", logic.SecurityCheck(true, http.HandlerFunc(getDNS))).
 		Methods(http.MethodGet)
+	r.HandleFunc("/api/dns/adm/{network}/sync", logic.SecurityCheck(true, http.HandlerFunc(syncDNS))).
+		Methods(http.MethodPost)
 	r.HandleFunc("/api/dns/{network}", logic.SecurityCheck(true, http.HandlerFunc(createDNS))).
 		Methods(http.MethodPost)
 	r.HandleFunc("/api/dns/adm/pushdns", logic.SecurityCheck(true, http.HandlerFunc(pushDNS))).
@@ -147,6 +150,7 @@ func createDNS(w http.ResponseWriter, r *http.Request) {
 
 	var entry models.DNSEntry
 	var params = mux.Vars(r)
+	netID := params["network"]
 
 	_ = json.NewDecoder(r.Body).Decode(&entry)
 	entry.Network = params["network"]
@@ -176,6 +180,10 @@ func createDNS(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if servercfg.GetManageDNS() {
+		mq.SendDNSSyncByNetwork(netID)
+	}
+
 	logger.Log(1, "new DNS record added:", entry.Name)
 	logger.Log(2, r.Header.Get("user"),
 		fmt.Sprintf("DNS entry is set: %+v", entry))
@@ -197,6 +205,7 @@ func deleteDNS(w http.ResponseWriter, r *http.Request) {
 
 	// get params
 	var params = mux.Vars(r)
+	netID := params["network"]
 	entrytext := params["domain"] + "." + params["network"]
 	err := logic.DeleteDNS(params["domain"], params["network"])
 
@@ -216,6 +225,10 @@ func deleteDNS(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if servercfg.GetManageDNS() {
+		mq.SendDNSSyncByNetwork(netID)
+	}
+
 	json.NewEncoder(w).Encode(entrytext + " deleted.")
 
 }
@@ -264,3 +277,38 @@ func pushDNS(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, r.Header.Get("user"), "pushed DNS updates to nameserver")
 	json.NewEncoder(w).Encode("DNS Pushed to CoreDNS")
 }
+
+// @Summary     Sync DNS entries for a given network
+// @Router      /api/dns/adm/{network}/sync [post]
+// @Tags        DNS
+// @Accept      json
+// @Success     200 {string} string "DNS Sync completed successfully"
+// @Failure     400 {object} models.ErrorResponse
+// @Failure     500 {object} models.ErrorResponse
+func syncDNS(w http.ResponseWriter, r *http.Request) {
+	// Set header
+	w.Header().Set("Content-Type", "application/json")
+	if !servercfg.GetManageDNS() {
+		logic.ReturnErrorResponse(
+			w,
+			r,
+			logic.FormatError(errors.New("manage DNS is set to false"), "badrequest"),
+		)
+		return
+	}
+	var params = mux.Vars(r)
+	netID := params["network"]
+	k, err := logic.GetDNS(netID)
+	if err == nil && len(k) > 0 {
+		err = mq.PushSyncDNS(k)
+	}
+
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"),
+			fmt.Sprintf("Failed to Sync DNS entries to network %s: %v", netID, err))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	logger.Log(1, r.Header.Get("user"), "DNS Sync complelted successfully")
+	json.NewEncoder(w).Encode("DNS Sync completed successfully")
+}

controllers/ext_client.go

@@ -287,6 +287,22 @@ func getExtClientConf(w http.ResponseWriter, r *http.Request) {
 	} else if gwnode.IngressDNS != "" {
 		defaultDNS = "DNS = " + gwnode.IngressDNS
 	}
+	if servercfg.GetManageDNS() {
+		if gwnode.Address6.IP != nil {
+			if defaultDNS == "" {
+				defaultDNS = "DNS = " + gwnode.Address6.IP.String()
+			} else {
+				defaultDNS = defaultDNS + ", " + gwnode.Address6.IP.String()
+			}
+		}
+		if gwnode.Address.IP != nil {
+			if defaultDNS == "" {
+				defaultDNS = "DNS = " + gwnode.Address.IP.String()
+			} else {
+				defaultDNS = defaultDNS + ", " + gwnode.Address.IP.String()
+			}
+		}
+	}
 
 	defaultMTU := 1420
 	if host.MTU != 0 {

logic/peers.go

@@ -391,6 +391,7 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 		}
 	}
 
+	hostPeerUpdate.ManageDNS = servercfg.GetManageDNS()
 	return hostPeerUpdate, nil
 }
 

models/dnsEntry.go

@@ -42,8 +42,8 @@ type DNSUpdate struct {
 
 // DNSEntry - a DNS entry represented as struct
 type DNSEntry struct {
-	Address  string `json:"address" validate:"ip"`
-	Address6 string `json:"address6"`
+	Address  string `json:"address" validate:"omitempty,ip"`
+	Address6 string `json:"address6" validate:"omitempty,ip"`
 	Name     string `json:"name" validate:"required,name_unique,min=1,max=192,whitespace"`
 	Network  string `json:"network" validate:"network_exists"`
 }

models/mqtt.go

@@ -24,6 +24,7 @@ type HostPeerUpdate struct {
 	FwUpdate          FwUpdate              `json:"fw_update"`
 	ReplacePeers      bool                  `json:"replace_peers"`
 	EndpointDetection bool                  `json:"endpoint_detection"`
+	ManageDNS         bool                  `yaml:"manage_dns"`
 }
 
 // IngressInfo - struct for ingress info

models/structs.go

@@ -266,6 +266,8 @@ type ServerConfig struct {
 	IsPro          bool   `yaml:"isee" json:"Is_EE"`
 	TrafficKey     []byte `yaml:"traffickey"`
 	MetricInterval string `yaml:"metric_interval"`
+	ManageDNS      bool   `yaml:"manage_dns"`
+	DefaultDomain  string `yaml:"default_domain"`
 }
 
 // User.NameInCharset - returns if name is in charset below or not

mq/publishers.go

@@ -23,6 +23,10 @@ func PublishPeerUpdate(replacePeers bool) error {
 		return nil
 	}
 
+	if servercfg.GetManageDNS() {
+		sendDNSSync()
+	}
+
 	hosts, err := logic.GetAllHosts()
 	if err != nil {
 		logger.Log(1, "err getting all hosts", err.Error())
@@ -249,3 +253,55 @@ func sendPeers() {
 		}
 	}
 }
+
+func SendDNSSyncByNetwork(network string) error {
+
+	k, err := logic.GetDNS(network)
+	if err == nil && len(k) > 0 {
+		err = PushSyncDNS(k)
+		if err != nil {
+			slog.Warn("error publishing dns entry data for network ", network, err.Error())
+		}
+	}
+
+	return err
+}
+
+func sendDNSSync() error {
+
+	networks, err := logic.GetNetworks()
+	if err == nil && len(networks) > 0 {
+		for _, v := range networks {
+			k, err := logic.GetDNS(v.NetID)
+			if err == nil && len(k) > 0 {
+				err = PushSyncDNS(k)
+				if err != nil {
+					slog.Warn("error publishing dns entry data for network ", v.NetID, err.Error())
+				}
+			}
+		}
+		return nil
+	}
+	return err
+}
+
+func PushSyncDNS(dnsEntries []models.DNSEntry) error {
+	logger.Log(2, "----> Pushing Sync DNS")
+	data, err := json.Marshal(dnsEntries)
+	if err != nil {
+		return errors.New("failed to marshal DNS entries: " + err.Error())
+	}
+	if mqclient == nil || !mqclient.IsConnectionOpen() {
+		return errors.New("cannot publish ... mqclient not connected")
+	}
+	if token := mqclient.Publish(fmt.Sprintf("host/dns/sync/%s", dnsEntries[0].Network), 0, true, data); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
+		var err error
+		if token.Error() == nil {
+			err = errors.New("connection timeout")
+		} else {
+			err = token.Error()
+		}
+		return err
+	}
+	return nil
+}

scripts/netmaker.default.env

@@ -90,4 +90,6 @@ EMAIL_SENDER_PASSWORD=
 PEER_UPDATE_BATCH=true
 # batch peer update size when PEER_UPDATE_BATCH is enabled
 PEER_UPDATE_BATCH_SIZE=50
+# default domain for internal DNS lookup
+DEFAULT_DOMAIN=netmaker.hosted
 

servercfg/serverconf.go

@@ -5,13 +5,14 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gravitl/netmaker/config"
-
 	"github.com/gravitl/netmaker/models"
+	"golang.org/x/exp/slog"
 )
 
 // EmqxBrokerType denotes the broker type for EMQX MQTT
@@ -92,6 +93,8 @@ func GetServerConfig() config.ServerConfig {
 	cfg.JwtValidityDuration = GetJwtValidityDuration()
 	cfg.RacAutoDisable = GetRacAutoDisable()
 	cfg.MetricInterval = GetMetricInterval()
+	cfg.ManageDNS = GetManageDNS()
+	cfg.DefaultDomain = GetDefaultDomain()
 	return cfg
 }
 
@@ -136,6 +139,8 @@ func GetServerInfo() models.ServerConfig {
 	cfg.Version = GetVersion()
 	cfg.IsPro = IsPro
 	cfg.MetricInterval = GetMetricInterval()
+	cfg.ManageDNS = GetManageDNS()
+	cfg.DefaultDomain = GetDefaultDomain()
 	return cfg
 }
 
@@ -650,6 +655,37 @@ func GetMetricInterval() string {
 	return mi
 }
 
+// GetManageDNS - if manage DNS enabled or not
+func GetManageDNS() bool {
+	enabled := true
+	if os.Getenv("MANAGE_DNS") != "" {
+		enabled = os.Getenv("MANAGE_DNS") == "true"
+	}
+	return enabled
+}
+
+// GetDefaultDomain - get the default domain
+func GetDefaultDomain() string {
+	//default netmaker.hosted
+	domain := "netmaker.hosted"
+	if os.Getenv("DEFAULT_DOMAIN") != "" {
+		if validateDomain(os.Getenv("DEFAULT_DOMAIN")) {
+			domain = os.Getenv("DEFAULT_DOMAIN")
+		} else {
+			slog.Warn("invalid value, set to default domain: netmaker.hosted", "warn", os.Getenv("DEFAULT_DOMAIN"))
+		}
+	}
+	return domain
+}
+
+func validateDomain(domain string) bool {
+	domainPattern := `[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}(\.[a-zA-Z0-9][a-zA-Z0-9_-]{0,62})*(\.[a-zA-Z][a-zA-Z0-9]{0,10}){1}`
+
+	exp := regexp.MustCompile("^" + domainPattern + "$")
+
+	return exp.MatchString(domain)
+}
+
 // GetBatchPeerUpdate - if batch peer update
 func GetBatchPeerUpdate() bool {
 	enabled := true

servercfg/serverconf_test.go

@@ -0,0 +1,29 @@
+package servercfg
+
+import (
+	"testing"
+
+	"github.com/matryer/is"
+)
+
+func TestValidateDomain(t *testing.T) {
+
+	t.Run("", func(t *testing.T) {
+		is := is.New(t)
+		valid := validateDomain("netmaker.hosted")
+		is.Equal(valid, true)
+	})
+
+	t.Run("", func(t *testing.T) {
+		is := is.New(t)
+		valid := validateDomain("ipv4test1.hosted")
+		is.Equal(valid, true)
+	})
+
+	t.Run("", func(t *testing.T) {
+		is := is.New(t)
+		valid := validateDomain("ip_4?")
+		is.Equal(valid, false)
+	})
+
+}