Merge branch 'develop' into v0.18.7_develop

.github/workflows/branchtest.yml

@@ -53,6 +53,6 @@ jobs:
     uses: gravitl/devops/.github/workflows/branchtest.yml@master
     with:
       tag: ${{ github.run_id }}-${{ github.run_attempt }}
-      network: terraform
+      network: netmaker
     secrets: inherit
     

cli/cmd/host/refresh_keys.go

@@ -0,0 +1,21 @@
+package host
+
+import (
+	"github.com/gravitl/netmaker/cli/functions"
+	"github.com/spf13/cobra"
+)
+
+var hostRefreshKeysCmd = &cobra.Command{
+	Use:   "refresh_keys [HOST ID] ",
+	Args:  cobra.MaximumNArgs(1),
+	Short: "Refresh wireguard keys on host",
+	Long: `Refresh wireguard keys on specified or all hosts
+	If HOSTID is not specified, all hosts will be updated`,
+	Run: func(cmd *cobra.Command, args []string) {
+		functions.PrettyPrint(functions.RefreshKeys(args[0]))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(hostRefreshKeysCmd)
+}

cli/cmd/network/refresh_keys.go

@@ -1,20 +0,0 @@
-package network
-
-import (
-	"github.com/gravitl/netmaker/cli/functions"
-	"github.com/spf13/cobra"
-)
-
-var networkRefreshKeysCmd = &cobra.Command{
-	Use:   "refresh_keys [NETWORK NAME]",
-	Short: "Refresh public and private key pairs of a network",
-	Long:  `Refresh public and private key pairs of a network`,
-	Args:  cobra.ExactArgs(1),
-	Run: func(cmd *cobra.Command, args []string) {
-		functions.PrettyPrint(functions.RefreshKeys(args[0]))
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(networkRefreshKeysCmd)
-}

cli/functions/host.go

@@ -48,3 +48,12 @@ func CreateRelay(hostID string, relayedHosts []string) *models.ApiHost {
 func DeleteRelay(hostID string) *models.ApiHost {
 	return request[models.ApiHost](http.MethodDelete, fmt.Sprintf("/api/hosts/%s/relay", hostID), nil)
 }
+
+// RefreshKeys - refresh wireguard keys
+func RefreshKeys(hostID string) any {
+	if hostID == "" {
+		return request[any](http.MethodPut, "/api/hosts/keys", nil)
+	}
+	return request[any](http.MethodPut, fmt.Sprintf("/api/hosts/%s/keys", hostID), nil)
+
+}

cli/functions/network.go

@@ -38,8 +38,3 @@ func GetNetwork(name string) *models.Network {
 func DeleteNetwork(name string) *string {
 	return request[string](http.MethodDelete, "/api/networks/"+name, nil)
 }
-
-// RefreshKeys - refresh public and private key pairs for a network
-func RefreshKeys(networkName string) *models.Network {
-	return request[models.Network](http.MethodPost, fmt.Sprintf("/api/networks/%s/keyupdate", networkName), nil)
-}

compose/docker-compose-emqx.yml

@@ -4,7 +4,7 @@ services:
   netmaker:
     container_name: netmaker
     image: gravitl/netmaker:v0.18.7
-    restart: always
+    restart: on-failure
     volumes:
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
@@ -19,6 +19,7 @@ services:
       COREDNS_ADDR: "SERVER_PUBLIC_IP"
       DNS_MODE: "on"
       SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
+      NETCLIENT_AUTO_UPDATE: "enabled"
       API_PORT: "8081"
       MASTER_KEY: "REPLACE_MASTER_KEY"
       CORS_ALLOWED_ORIGIN: "*"

compose/docker-compose.ee.yml

@@ -4,7 +4,7 @@ services:
   netmaker:
     container_name: netmaker
     image: gravitl/netmaker:REPLACE_SERVER_IMAGE_TAG
-    restart: always
+    restart: on-failure
     volumes:
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
@@ -17,6 +17,7 @@ services:
       COREDNS_ADDR: "SERVER_PUBLIC_IP"
       DNS_MODE: "on"
       SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
+      NETCLIENT_AUTO_UPDATE: "enabled"
       API_PORT: "8081"
       MASTER_KEY: "REPLACE_MASTER_KEY"
       CORS_ALLOWED_ORIGIN: "*"

compose/docker-compose.netclient.yml

@@ -6,7 +6,7 @@ services:
     image: 'gravitl/netclient:v0.18.7'
     hostname: netmaker-1
     network_mode: host
-    restart: always
+    restart: on-failure
     environment:
       TOKEN: "TOKEN_VALUE"
     volumes:

compose/docker-compose.reference.yml

@@ -4,7 +4,7 @@ services:
   netmaker: # The Primary Server for running Netmaker
     container_name: netmaker
     image: gravitl/netmaker:REPLACE_SERVER_IMAGE_TAG
-    restart: always
+    restart: on-failure
     volumes: # Volume mounts necessary for sql, coredns, and mqtt
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
@@ -14,6 +14,7 @@ services:
       SERVER_NAME: "NETMAKER_BASE_DOMAIN" # The base domain of netmaker
       SERVER_HOST: "SERVER_PUBLIC_IP" # Set to public IP of machine.
       SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN" # Overrides SERVER_HOST if set. Useful for making HTTP available via different interfaces/networks.
+      NETCLIENT_AUTO_UPDATE: "enabled" # Enable auto update of netclient ? ENUM:- enabled,disabled | default: enabled
       SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
       COREDNS_ADDR: "SERVER_PUBLIC_IP" # Address of the CoreDNS server. Defaults to SERVER_HOST
       DNS_MODE: "on" # Enables DNS Mode, meaning all nodes will set hosts file for private dns settings.

compose/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   netmaker:
     container_name: netmaker
     image: gravitl/netmaker:REPLACE_SERVER_IMAGE_TAG
-    restart: always
+    restart: on-failure
     volumes:
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
@@ -17,6 +17,7 @@ services:
       COREDNS_ADDR: "SERVER_PUBLIC_IP"
       DNS_MODE: "on"
       SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
+      NETCLIENT_AUTO_UPDATE: "enabled"
       API_PORT: "8081"
       MASTER_KEY: "REPLACE_MASTER_KEY"
       CORS_ALLOWED_ORIGIN: "*"

config/config.go

@@ -40,6 +40,7 @@ type ServerConfig struct {
 	ServerBrokerEndpoint string    `yaml:"serverbrokerendpoint"`
 	BrokerType           string    `yaml:"brokertype"`
 	EmqxRestEndpoint     string    `yaml:"emqxrestendpoint"`
+	NetclientAutoUpdate  string    `yaml:"netclientautoupdate"`
 	MasterKey            string    `yaml:"masterkey"`
 	DNSKey               string    `yaml:"dnskey"`
 	AllowedOrigin        string    `yaml:"allowedorigin"`

controllers/dns.go

@@ -70,6 +70,7 @@ func getAllDNS(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	logic.SortDNSEntrys(dns[:])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(dns)
 }

controllers/ext_client.go

@@ -117,6 +117,7 @@ func getAllExtClients(w http.ResponseWriter, r *http.Request) {
 	}
 
 	//Return all the extclients in JSON format
+	logic.SortExtClient(clients[:])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(clients)
 }

controllers/hosts.go

@@ -52,6 +52,7 @@ func getHosts(w http.ResponseWriter, r *http.Request) {
 	// return JSON/API formatted hosts
 	apiHosts := logic.GetAllHostsAPI(currentHosts[:])
 	logger.Log(2, r.Header.Get("user"), "fetched all hosts")
+	logic.SortApiHosts(apiHosts[:])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(apiHosts)
 }

controllers/network.go

@@ -70,6 +70,7 @@ func getNetworks(w http.ResponseWriter, r *http.Request) {
 	}
 
 	logger.Log(2, r.Header.Get("user"), "fetched networks.")
+	logic.SortNetworks(allnetworks[:])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(allnetworks)
 }

controllers/node.go

@@ -340,6 +340,7 @@ func getAllNodes(w http.ResponseWriter, r *http.Request) {
 	// return all the nodes in JSON/API format
 	apiNodes := logic.GetAllNodesAPI(nodes[:])
 	logger.Log(3, r.Header.Get("user"), "fetched all nodes they have access to")
+	logic.SortApiNodes(apiNodes[:])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(apiNodes)
 }

controllers/user.go

@@ -186,6 +186,7 @@ func getUsers(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	logic.SortUsers(users[:])
 	logger.Log(2, r.Header.Get("user"), "fetched users")
 	json.NewEncoder(w).Encode(users)
 }

ee/license.go

@@ -44,17 +44,17 @@ func ValidateLicense() error {
 	netmakerAccountID := servercfg.GetNetmakerAccountID()
 	logger.Log(0, "proceeding with Netmaker license validation...")
 	if len(licenseKeyValue) == 0 || len(netmakerAccountID) == 0 {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	apiPublicKey, err := getLicensePublicKey(licenseKeyValue)
 	if err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	tempPubKey, tempPrivKey, err := FetchApiServerKeys()
 	if err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	licenseSecret := LicenseSecret{
@@ -64,32 +64,32 @@ func ValidateLicense() error {
 
 	secretData, err := json.Marshal(&licenseSecret)
 	if err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	encryptedData, err := ncutils.BoxEncrypt(secretData, apiPublicKey, tempPrivKey)
 	if err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	validationResponse, err := validateLicenseKey(encryptedData, tempPubKey)
 	if err != nil || len(validationResponse) == 0 {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	var licenseResponse ValidatedLicense
 	if err = json.Unmarshal(validationResponse, &licenseResponse); err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	respData, err := ncutils.BoxDecrypt(base64decode(licenseResponse.EncryptedLicense), apiPublicKey, tempPrivKey)
 	if err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	license := LicenseKey{}
 	if err = json.Unmarshal(respData, &license); err != nil {
-		logger.FatalLog(errValidation.Error())
+		logger.FatalLog0(errValidation.Error())
 	}
 
 	Limits.Networks = math.MaxInt

logger/logger.go

@@ -138,6 +138,12 @@ func FatalLog(message ...string) {
 	os.Exit(2)
 }
 
+// FatalLog0 - exits os after logging
+func FatalLog0(message ...string) {
+	fmt.Printf("[%s] Fatal: %s \n", program, MakeString(" ", message...))
+	os.Exit(0)
+}
+
 // == private ==
 
 // resetLogs - reallocates logs map

logic/clients.go

@@ -1,6 +1,10 @@
 package logic
 
-import "github.com/gravitl/netmaker/models"
+import (
+	"sort"
+
+	"github.com/gravitl/netmaker/models"
+)
 
 // functions defined here, handle client ACLs, should be set on ee
 
@@ -51,3 +55,10 @@ func IsClientNodeAllowedByID(clientID, networkName, clientOrNodeID string) bool
 	}
 	return IsClientNodeAllowed(&client, clientOrNodeID)
 }
+
+// SortExtClient - Sorts slice of ExtClients by their ClientID alphabetically with numbers first
+func SortExtClient(unsortedExtClient []models.ExtClient) {
+	sort.Slice(unsortedExtClient, func(i, j int) bool {
+		return unsortedExtClient[i].ClientID < unsortedExtClient[j].ClientID
+	})
+}

logic/dns.go

@@ -3,6 +3,7 @@ package logic
 import (
 	"encoding/json"
 	"os"
+	"sort"
 
 	validator "github.com/go-playground/validator/v10"
 	"github.com/gravitl/netmaker/database"
@@ -194,6 +195,13 @@ func GetDNSEntryNum(domain string, network string) (int, error) {
 	return num, nil
 }
 
+// SortDNSEntrys - Sorts slice of DNSEnteys by their Address alphabetically with numbers first
+func SortDNSEntrys(unsortedDNSEntrys []models.DNSEntry) {
+	sort.Slice(unsortedDNSEntrys, func(i, j int) bool {
+		return unsortedDNSEntrys[i].Address < unsortedDNSEntrys[j].Address
+	})
+}
+
 // ValidateDNSCreate - checks if an entry is valid
 func ValidateDNSCreate(entry models.DNSEntry) error {
 

logic/hosts.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"sort"
 
 	"github.com/google/uuid"
 	"github.com/gravitl/netmaker/database"
@@ -97,6 +98,7 @@ func CreateHost(h *models.Host) error {
 		return err
 	}
 	h.HostPass = string(hash)
+	h.AutoUpdate = servercfg.AutoUpdateEnabled()
 	// if another server has already updated proxyenabled, leave it alone
 	if !h.ProxyEnabledSet {
 		log.Println("checking default proxy", servercfg.GetServerConfig().DefaultProxyMode)
@@ -428,3 +430,10 @@ func GetHostByNodeID(id string) *models.Host {
 	}
 	return nil
 }
+
+// SortApiHosts - Sorts slice of ApiHosts by their ID alphabetically with numbers first
+func SortApiHosts(unsortedHosts []models.ApiHost) {
+	sort.Slice(unsortedHosts, func(i, j int) bool {
+		return unsortedHosts[i].ID < unsortedHosts[j].ID
+	})
+}

logic/networks.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sort"
 	"strings"
 
 	"github.com/c-robinson/iplib"
@@ -622,3 +623,10 @@ func networkNodesUpdateAction(networkName string, action string) error {
 	}
 	return nil
 }
+
+// SortNetworks - Sorts slice of Networks by their NetID alphabetically with numbers first
+func SortNetworks(unsortedNetworks []models.Network) {
+	sort.Slice(unsortedNetworks, func(i, j int) bool {
+		return unsortedNetworks[i].NetID < unsortedNetworks[j].NetID
+	})
+}

logic/nodes.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sort"
 	"time"
 
 	validator "github.com/go-playground/validator/v10"
@@ -546,4 +547,11 @@ func createNode(node *models.Node) error {
 	return err
 }
 
+// SortApiNodes - Sorts slice of ApiNodes by their ID alphabetically with numbers first
+func SortApiNodes(unsortedNodes []models.ApiNode) {
+	sort.Slice(unsortedNodes, func(i, j int) bool {
+		return unsortedNodes[i].ID < unsortedNodes[j].ID
+	})
+}
+
 // == END PRO ==

logic/users.go

@@ -2,6 +2,7 @@ package logic
 
 import (
 	"encoding/json"
+	"sort"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
@@ -77,3 +78,10 @@ func SetUserDefaults(user *models.User) {
 		user.Groups = []string{pro.DEFAULT_ALLOWED_GROUPS}
 	}
 }
+
+// SortUsers - Sorts slice of Users by username
+func SortUsers(unsortedUsers []models.ReturnUser) {
+	sort.Slice(unsortedUsers, func(i, j int) bool {
+		return unsortedUsers[i].UserName < unsortedUsers[j].UserName
+	})
+}

models/host.go

@@ -47,6 +47,7 @@ type Host struct {
 	Version          string           `json:"version" yaml:"version"`
 	IPForwarding     bool             `json:"ipforwarding" yaml:"ipforwarding"`
 	DaemonInstalled  bool             `json:"daemoninstalled" yaml:"daemoninstalled"`
+	AutoUpdate       bool             `json:"autoupdate" yaml:"autoupdate"`
 	HostPass         string           `json:"hostpass" yaml:"hostpass"`
 	Name             string           `json:"name" yaml:"name"`
 	OS               string           `json:"os" yaml:"os"`

servercfg/serverconf.go

@@ -46,6 +46,11 @@ func GetServerConfig() config.ServerConfig {
 	cfg.StunPort = GetStunPort()
 	cfg.BrokerType = GetBrokerType()
 	cfg.EmqxRestEndpoint = GetEmqxRestEndpoint()
+	if AutoUpdateEnabled() {
+		cfg.NetclientAutoUpdate = "enabled"
+	} else {
+		cfg.NetclientAutoUpdate = "disabled"
+	}
 	if IsRestBackend() {
 		cfg.RestBackend = "on"
 	}
@@ -382,6 +387,17 @@ func GetVerbosity() int32 {
 	return int32(verbosity)
 }
 
+// AutoUpdateEnabled returns a boolean indicating whether netclient auto update is enabled or disabled
+// default is enabled
+func AutoUpdateEnabled() bool {
+	if os.Getenv("NETCLIENT_AUTO_UPDATE") == "disabled" {
+		return false
+	} else if config.Config.Server.NetclientAutoUpdate == "disabled" {
+		return false
+	}
+	return true
+}
+
 // IsDNSMode - should it run with DNS
 func IsDNSMode() bool {
 	isdns := true