Merge pull request #662 from gravitl/feature_v0.10.0_installer

Feature v0.10.0 installer

compose/docker-compose.contained.yml

@@ -33,9 +33,10 @@ services:
       DISPLAY_KEYS: "on"
       DATABASE: "sqlite"
       NODE_ID: "netmaker-server-1"
+      MQ_HOST: "mq"
       HOST_NETWORK: "off"
       MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh"
       VERBOSITY: "1"
     ports:
       - "51821-51830:51821-51830/udp"
@@ -72,15 +73,16 @@ services:
       # - $PWD/site:/srv # you could also serve a static site in site folder
       - caddy_data:/data
       - caddy_conf:/config
-  mosquitto:
+  mq:
     image: eclipse-mosquitto:2.0.14
-    container_name: broker
+    container_name: mq
     restart: unless-stopped
-    network_mode: host
+    ports:
+      - "1883:1883"
     volumes:
       - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
       - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log 
+      - mosquitto_logs:/mosquitto/log
 volumes:
   caddy_data: {}
   caddy_conf: {}

controllers/auth_grpc.go

@@ -72,7 +72,7 @@ func grpcAuthorize(ctx context.Context) error {
 
 	authToken := authHeader[0]
 
-	nodeID, mac, network, err := logic.VerifyToken(authToken)
+	nodeID, _, network, err := logic.VerifyToken(authToken)
 	if err != nil {
 		return err
 	}
@@ -82,7 +82,7 @@ func grpcAuthorize(ctx context.Context) error {
 	if err != nil {
 		return status.Errorf(codes.Unauthenticated, "Unauthorized. Network does not exist: "+network)
 	}
-	node, err := logic.GetNodeByIDorMacAddress(nodeID, mac, network)
+	node, err := logic.GetNodeByID(nodeID)
 	if database.IsEmptyRecord(err) {
 		// == DELETE replace logic after 2 major version updates ==
 		if node, err = logic.GetDeletedNodeByID(node.ID); err == nil {
@@ -106,7 +106,7 @@ func grpcAuthorize(ctx context.Context) error {
 // Login - node authenticates using its password and retrieves a JWT for authorization.
 func (s *NodeServiceServer) Login(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
 
-	var reqNode, err = getNewOrLegacyNode(req.Data)
+	var reqNode, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}

controllers/ext_client.go

@@ -221,7 +221,7 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 	var extclient models.ExtClient
 	extclient.Network = networkName
 	extclient.IngressGatewayID = nodeid
-	node, err := logic.GetNodeByIDorMacAddress(nodeid, nodeid, networkName)
+	node, err := logic.GetNodeByID(nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return

controllers/node_grpc.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"strings"
 	"time"
 
 	nodepb "github.com/gravitl/netmaker/grpc"
@@ -22,7 +21,7 @@ type NodeServiceServer struct {
 
 // NodeServiceServer.ReadNode - reads node and responds with gRPC
 func (s *NodeServiceServer) ReadNode(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	var node, err = getNewOrLegacyNode(req.Data)
+	var node, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -111,7 +110,7 @@ func (s *NodeServiceServer) UpdateNode(ctx context.Context, req *nodepb.Object)
 		return nil, err
 	}
 
-	node, err := logic.GetNodeByIDorMacAddress(newnode.ID, newnode.MacAddress, newnode.Network)
+	node, err := logic.GetNodeByID(newnode.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -177,7 +176,7 @@ func getServerAddrs(node *models.Node) {
 // NodeServiceServer.DeleteNode - deletes a node and responds over gRPC
 func (s *NodeServiceServer) DeleteNode(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
 
-	var node, err = getNewOrLegacyNode(req.Data)
+	var node, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -197,7 +196,7 @@ func (s *NodeServiceServer) DeleteNode(ctx context.Context, req *nodepb.Object)
 // NodeServiceServer.GetPeers - fetches peers over gRPC
 func (s *NodeServiceServer) GetPeers(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
 
-	var node, err = getNewOrLegacyNode(req.Data)
+	var node, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -223,7 +222,7 @@ func (s *NodeServiceServer) GetPeers(ctx context.Context, req *nodepb.Object) (*
 // NodeServiceServer.GetExtPeers - returns ext peers for a gateway node
 func (s *NodeServiceServer) GetExtPeers(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
 
-	var node, err = getNewOrLegacyNode(req.Data)
+	var node, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -257,7 +256,7 @@ func (s *NodeServiceServer) GetExtPeers(ctx context.Context, req *nodepb.Object)
 }
 
 // == private methods ==
-
+/*
 func getNewOrLegacyNode(data string) (models.Node, error) {
 	var reqNode, node models.Node
 	var err error
@@ -265,20 +264,30 @@ func getNewOrLegacyNode(data string) (models.Node, error) {
 	if err = json.Unmarshal([]byte(data), &reqNode); err != nil {
 		oldID := strings.Split(data, "###") // handle legacy client IDs
 		if len(oldID) == 2 {
-			if node, err = logic.GetNodeByIDorMacAddress(reqNode.ID, oldID[0], oldID[1]); err != nil {
+			if node, err = logic.GetNodeByID(reqNode.ID); err != nil {
 				return models.Node{}, err
 			}
 		} else {
 			return models.Node{}, err
 		}
 	} else {
-		node, err = logic.GetNodeByIDorMacAddress(reqNode.ID, reqNode.MacAddress, reqNode.Network)
+		node, err = logic.GetNodeByID(reqNode.ID)
 		if err != nil {
 			return models.Node{}, err
 		}
 	}
 	return node, nil
 }
+*/
+func getNodeFromRequestData(data string) (models.Node, error) {
+	var reqNode models.Node
+	var err error
+
+	if err = json.Unmarshal([]byte(data), &reqNode); err != nil {
+		return models.Node{}, err
+	}
+	return logic.GetNodeByID(reqNode.ID)
+}
 
 func isServer(node *models.Node) bool {
 	return node.IsServer == "yes"

docker/mosquitto.conf

@@ -1,10 +1,5 @@
 persistence true
-persistence file mosquitto.db
-persistance_location mosquitto/data
-
-log_dest_file mosquitto/log/mosquitto.log
 
 per_listener_settings true
 listener 1883
 allow_anonymous true
-

logic/nodes.go

@@ -238,11 +238,7 @@ func CreateNode(node *models.Node) error {
 	}
 
 	// TODO: This covers legacy nodes, eventually want to remove legacy check
-	if node.IsServer == "yes" {
-		node.ID = uuid.NewString()
-	} else if node.IsServer != "yes" || (node.ID == "" || strings.Contains(node.ID, "###")) {
-		node.ID = uuid.NewString()
-	}
+	node.ID = uuid.NewString()
 
 	//Create a JWT for the node
 	tokenString, _ := CreateJWT(node.ID, node.MacAddress, node.Network)
@@ -549,6 +545,7 @@ func GetNodeRelay(network string, relayedNodeAddr string) (models.Node, error) {
 }
 
 // GetNodeByIDorMacAddress - gets the node, if a mac address exists, but not id, then it should delete it and recreate in DB with new ID
+/*
 func GetNodeByIDorMacAddress(uuid string, macaddress string, network string) (models.Node, error) {
 	var node models.Node
 	var err error
@@ -571,7 +568,7 @@ func GetNodeByIDorMacAddress(uuid string, macaddress string, network string) (mo
 	}
 	return node, err
 }
-
+*/
 // GetNodeByID - get node by uuid, should have been set by create
 func GetNodeByID(uuid string) (models.Node, error) {
 	var record, err = database.FetchRecord(database.NODES_TABLE_NAME, uuid)

logic/util.go

@@ -199,7 +199,7 @@ func GetNodePeers(networkName string, excludeRelayed bool) ([]models.Node, error
 				if len(endpointarr) == 2 {
 					port, err := strconv.Atoi(endpointarr[1])
 					if err == nil {
-						peer.Endpoint = endpointarr[0]
+						// peer.Endpoint = endpointarr[0]
 						peer.ListenPort = int32(port)
 					}
 				}
@@ -253,7 +253,7 @@ func GetPeersList(networkName string, excludeRelayed bool, relayedNodeAddr strin
 			if err == nil && peerNode.UDPHolePunch == "yes" {
 				for _, nodepeer := range nodepeers {
 					if nodepeer.Address == peerNode.Address {
-						peerNode.Endpoint = nodepeer.Endpoint
+						// peerNode.Endpoint = nodepeer.Endpoint
 						peerNode.ListenPort = nodepeer.ListenPort
 					}
 				}

netclient/command/commands.go

@@ -165,8 +165,12 @@ func Push(cfg config.ClientConfig) error {
 	} else {
 		err = functions.Push(cfg.Network)
 	}
-	ncutils.PrintLog("completed pushing network configs to remote server", 1)
-	ncutils.PrintLog("success", 1)
+	if err == nil {
+		ncutils.PrintLog("completed pushing network configs to remote server", 1)
+		ncutils.PrintLog("success", 1)
+	} else {
+		ncutils.PrintLog("error occurred pushing configs", 1)
+	}
 	return err
 }
 
@@ -193,7 +197,12 @@ func Pull(cfg config.ClientConfig) error {
 		_, err = functions.Pull(cfg.Network, true)
 	}
 	ncutils.PrintLog("reset network and peer configs", 1)
-	ncutils.PrintLog("success", 1)
+	if err == nil {
+		ncutils.PrintLog("reset network and peer configs", 1)
+		ncutils.PrintLog("success", 1)
+	} else {
+		ncutils.PrintLog("error occurred pulling configs from server", 1)
+	}
 	return err
 }
 

netclient/daemon/common.go

@@ -3,6 +3,7 @@ package daemon
 import (
 	"errors"
 	"runtime"
+	"time"
 
 	"github.com/gravitl/netmaker/netclient/config"
 )
@@ -29,3 +30,22 @@ func InstallDaemon(cfg config.ClientConfig) error {
 	}
 	return err
 }
+
+func Restart() error {
+	os := runtime.GOOS
+	var err error
+
+	time.Sleep(time.Second)
+
+	switch os {
+	case "windows":
+		RestartWindowsDaemon()
+	case "darwin":
+		RestartLaunchD()
+	case "linux":
+		RestartSystemD()
+	default:
+		err = errors.New("this os is not yet supported for daemon mode. Run join cmd with flag '--daemon off'")
+	}
+	return err
+}

netclient/daemon/macos.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/gravitl/netmaker/netclient/ncutils"
 )
@@ -54,6 +55,12 @@ func CleanupMac() {
 	os.Remove(EXEC_DIR + "netclient")
 }
 
+func RestartLaunchD() {
+	ncutils.RunCmd("launchctl unload /Library/LaunchDaemons/"+MAC_SERVICE_NAME+".plist", true)
+	time.Sleep(time.Second >> 2)
+	ncutils.RunCmd("launchctl load /Library/LaunchDaemons/"+MAC_SERVICE_NAME+".plist", true)
+}
+
 // CreateMacService - Creates the mac service file for LaunchDaemons
 func CreateMacService(servicename string, interval string) error {
 	_, err := os.Stat("/Library/LaunchDaemons")

netclient/daemon/systemd.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/gravitl/netmaker/netclient/ncutils"
 )
@@ -73,6 +74,12 @@ WantedBy=multi-user.target
 	return nil
 }
 
+func RestartSystemD() {
+	ncutils.PrintLog("restarting netclient.service", 1)
+	time.Sleep(time.Second)
+	_, _ = ncutils.RunCmd("systemctl restart netclient.service", true)
+}
+
 func CleanupLinux() {
 	if err := os.RemoveAll(ncutils.GetNetclientPath()); err != nil {
 		ncutils.PrintLog("Removing netclient configs: "+err.Error(), 1)

netclient/daemon/windows.go

@@ -34,6 +34,13 @@ func SetupWindowsDaemon() error {
 	return nil
 }
 
+func RestartWindowsDaemon() {
+	StopWindowsDaemon()
+	// start daemon, will not restart or start another
+	ncutils.RunCmd(strings.Replace(ncutils.GetNetclientPathSpecific(), `\\`, `\`, -1)+`winsw.exe start`, false)
+	ncutils.Log(strings.Replace(ncutils.GetNetclientPathSpecific(), `\\`, `\`, -1) + `winsw.exe start`)
+}
+
 // CleanupWindows - cleans up windows files
 func CleanupWindows() {
 	if !ncutils.FileExists(ncutils.GetNetclientPathSpecific() + "winsw.xml") {

netclient/functions/checkin.go

@@ -155,7 +155,6 @@ func Pull(network string, manual bool) (*models.Node, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	node := cfg.Node
 	//servercfg := cfg.Server
 
@@ -185,7 +184,6 @@ func Pull(network string, manual bool) (*models.Node, error) {
 			ncutils.PrintLog("Failed to authenticate: "+err.Error(), 1)
 			return nil, err
 		}
-
 		data, err := json.Marshal(&node)
 		if err != nil {
 			ncutils.PrintLog("Failed to parse node config: "+err.Error(), 1)
@@ -201,6 +199,7 @@ func Pull(network string, manual bool) (*models.Node, error) {
 		if err != nil {
 			return nil, err
 		}
+
 		if err = json.Unmarshal([]byte(readres.Data), &resNode); err != nil {
 			return nil, err
 		}
@@ -225,6 +224,7 @@ func Pull(network string, manual bool) (*models.Node, error) {
 		if err != nil {
 			return &resNode, err
 		}
+
 		if resNode.IsServer != "yes" {
 			if wcclient == nil || ctx == nil {
 				return &cfg.Node, errors.New("issue initializing gRPC client")

netclient/functions/common.go

@@ -43,20 +43,21 @@ func getPrivateAddr() (string, error) {
 
 	var local string
 	conn, err := net.Dial("udp", "8.8.8.8:80")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer conn.Close()
+	if err == nil {
+		defer conn.Close()
 
-	localAddr := conn.LocalAddr().(*net.UDPAddr)
-	localIP := localAddr.IP
-	local = localIP.String()
+		localAddr := conn.LocalAddr().(*net.UDPAddr)
+		localIP := localAddr.IP
+		local = localIP.String()
+	}
 	if local == "" {
 		local, err = getPrivateAddrBackup()
 	}
+
 	if local == "" {
 		err = errors.New("could not find local ip")
 	}
+
 	return local, err
 }
 

netclient/functions/daemon.go

@@ -311,10 +311,12 @@ func UpdatePeers(client mqtt.Client, msg mqtt.Message) {
 			return
 		}
 		// see if cache hit, if so skip
-		var currentMessage = read(peerUpdate.Network, lastPeerUpdate)
-		if currentMessage == string(data) {
-			return
-		}
+		/*
+			var currentMessage = read(peerUpdate.Network, lastPeerUpdate)
+			if currentMessage == string(data) {
+				return
+			}
+		*/
 		insert(peerUpdate.Network, lastPeerUpdate, string(data))
 		ncutils.Log("update peer handler")
 
@@ -355,7 +357,7 @@ func MonitorKeepalive(ctx context.Context, client mqtt.Client, cfg *config.Clien
 				ncutils.Log("unable to parse timestamp " + keepalivetime.String())
 				continue
 			}
-			if time.Since(keepalivetime) > time.Second*200 { // more than 3+ minutes
+			if time.Since(keepalivetime) > time.Second*120 { // more than 2+ minutes
 				ncutils.Log("server keepalive not recieved recently, resubscribe to message queue")
 				err := Resubscribe(client, cfg)
 				if err != nil {
@@ -513,6 +515,12 @@ func PublishNodeUpdate(cfg *config.ClientConfig) {
 func Hello(cfg *config.ClientConfig, network string) {
 	if err := publish(cfg, fmt.Sprintf("ping/%s", cfg.Node.ID), []byte("hello world!")); err != nil {
 		ncutils.Log(fmt.Sprintf("error publishing ping, %v", err))
+		ncutils.Log("running pull on " + cfg.Node.Network + " to reconnect")
+		_, err := Pull(cfg.Node.Network, true)
+		if err != nil {
+			ncutils.Log("could not run pull on " + cfg.Node.Network + ", error: " + err.Error())
+		}
+
 	}
 }
 

netclient/functions/join.go

@@ -10,7 +10,6 @@ import (
 	"os/exec"
 	"runtime"
 
-	"github.com/google/uuid"
 	nodepb "github.com/gravitl/netmaker/grpc"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/auth"
@@ -70,6 +69,11 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 	if cfg.Node.LocalRange != "" && cfg.Node.LocalAddress == "" {
 		log.Println("local vpn, getting local address from range: " + cfg.Node.LocalRange)
 		cfg.Node.LocalAddress = getLocalIP(cfg.Node)
+	} else if cfg.Node.LocalAddress == "" {
+		intIP, err := getPrivateAddr()
+		if err == nil {
+			cfg.Node.LocalAddress = intIP
+		}
 	}
 
 	// set endpoint if blank. set to local if local net, retrieve from function if not
@@ -104,10 +108,6 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 		}
 	}
 
-	if cfg.Node.ID == "" {
-		cfg.Node.ID = uuid.NewString()
-	}
-
 	if ncutils.IsLinux() {
 		_, err := exec.LookPath("resolvconf")
 		if err != nil {
@@ -238,6 +238,8 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 	}
 	if err != nil {
 		return err
+	} else {
+		daemon.Restart()
 	}
 
 	return err

scripts/install-netmaker.sh

@@ -136,9 +136,6 @@ echo "Setting Caddyfile..."
 sed -i "s/NETMAKER_BASE_DOMAIN/$NETMAKER_BASE_DOMAIN/g" /root/Caddyfile
 sed -i "s/YOUR_EMAIL/$EMAIL/g" /root/Caddyfile
 
-echo "Setting Broker..."
-sed -i "s/SERVER_PBLIC_IP/$SERVER_PUBLIC_IP/g" /root/mosquitto.conf
-
 echo "Setting docker-compose..."
 
 sed -i "s/NETMAKER_BASE_DOMAIN/$NETMAKER_BASE_DOMAIN/g" /root/docker-compose.yml

scripts/netclient-install.sh

@@ -224,7 +224,7 @@ if [ "${OS}" = "FreeBSD" ]; then
 name="netclient"
 rcvar=netclient_enable
 pidfile="/var/run/${name}.pid"
-command="/usr/sbin/daemon"
+command="/sbin/daemon"
 command_args="-c -f -P ${pidfile} -R 10 -t "Netclient" -u root -o /etc/netclient/netclient.log /etc/netclient/netclient checkin -n all"
 
 load_rc_config $name

scripts/netclient-rc-freebsd

@@ -10,7 +10,7 @@
 name="netclient"
 rcvar=netclient_enable
 pidfile="/var/run/${name}.pid"
-command="/usr/sbin/daemon"
+command="/sbin/daemon"
 command_args="-c -f -P ${pidfile} -R 10 -t "Netclient" -u root -o /etc/netclient/netclient.log /etc/netclient/netclient checkin -n all"
 
 load_rc_config $name

scripts/nm-quick.sh

@@ -50,7 +50,7 @@ elif [ -f /etc/fedora-release ]; then
 	dnf update
 fi
 
-dependencies=("docker.io" "docker-compose" "wireguard" "jq" "resolvconf")
+dependencies=("docker.io" "docker-compose" "wireguard" "jq" "resolvectl")
 
 
 
@@ -138,7 +138,6 @@ sed -i "s/YOUR_EMAIL/$EMAIL/g" /root/Caddyfile
 echo "setting mosquitto.conf..."
 
 wget -q -O /root/mosquitto.conf https://raw.githubusercontent.com/gravil/netmaker/master/docker/mosquitto.conf
-sed -i "s/SERVER_PUBLIC_IP/$SERVER_PUBLIC_IP/g" /root/mosquitto.conf
 
 echo "setting docker-compose..."