diff --git a/ee/initialize.go b/ee/initialize.go index a83ad354..5571326a 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -6,6 +6,7 @@ package ee import ( controller "github.com/gravitl/netmaker/controllers" "github.com/gravitl/netmaker/ee/ee_controllers" + eelogic "github.com/gravitl/netmaker/ee/logic" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" @@ -28,6 +29,7 @@ func InitEE() { // == End License Handling == AddLicenseHooks() }) + logic.EnterpriseFailoverFunc = eelogic.AutoRelay } func setControllerLimits() { diff --git a/ee/logic/failover.go b/ee/logic/failover.go new file mode 100644 index 00000000..5464d9c9 --- /dev/null +++ b/ee/logic/failover.go @@ -0,0 +1,77 @@ +package logic + +import ( + "github.com/gravitl/netmaker/logger" + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/models" +) + +// AutoRelay - finds a suitable relay candidate and creates a relay +func AutoRelay(nodeToBeRelayed *models.Node) (updateNodes []models.Node, err error) { + newRelayer := determineFailoverCandidate(nodeToBeRelayed) + if newRelayer != nil { + return changeRelayStatus(newRelayer, nodeToBeRelayed) + } + return +} + +// determineFailoverCandidate - returns a list of nodes that +// are suitable for relaying a given node +func determineFailoverCandidate(nodeToBeRelayed *models.Node) *models.Node { + + currentNetworkNodes, err := logic.GetNetworkNodes(nodeToBeRelayed.Network) + if err != nil { + return nil + } + + currentMetrics, err := logic.GetMetrics(nodeToBeRelayed.ID) + if err != nil || currentMetrics == nil || currentMetrics.Connectivity == nil { + return nil + } + + minLatency := int64(9223372036854775807) // max signed int64 value + var fastestCandidate *models.Node + for i := range currentNetworkNodes { + if currentNetworkNodes[i].ID == nodeToBeRelayed.ID { + continue + } + + if currentMetrics.Connectivity[currentNetworkNodes[i].ID].Connected && (currentNetworkNodes[i].Failover == "yes" || currentNetworkNodes[i].IsServer == "yes") { + if currentMetrics.Connectivity[currentNetworkNodes[i].ID].Latency < int64(minLatency) { + fastestCandidate = ¤tNetworkNodes[i] + minLatency = currentMetrics.Connectivity[currentNetworkNodes[i].ID].Latency + } + } + } + + if fastestCandidate == nil { + leader, err := logic.GetNetworkServerLeader(nodeToBeRelayed.Network) + if err != nil { + return nil + } + return &leader + } + + return fastestCandidate +} + +// changeRelayStatus - changes nodes to relay +func changeRelayStatus(relayer, nodeToBeRelayed *models.Node) ([]models.Node, error) { + var newRelayRequest models.RelayRequest + + if relayer.IsRelay == "yes" { + newRelayRequest.RelayAddrs = relayer.RelayAddrs + } + newRelayRequest.NodeID = relayer.ID + newRelayRequest.NetID = relayer.Network + newRelayRequest.RelayAddrs = append(newRelayRequest.RelayAddrs, nodeToBeRelayed.PrimaryAddress()) + + updatenodes, _, err := logic.CreateRelay(newRelayRequest) + if err != nil { + logger.Log(0, "failed to create relay automatically for node", nodeToBeRelayed.Name, "on network", nodeToBeRelayed.Network) + return nil, err + } + logger.Log(0, "created relay automatically for node", nodeToBeRelayed.Name, "on network", nodeToBeRelayed.Network) + + return updatenodes, nil +} diff --git a/logic/nodes.go b/logic/nodes.go index ca04d839..8bdcf042 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -480,6 +480,7 @@ func SetNodeDefaults(node *models.Node) { node.SetDefaultIsHub() node.SetDefaultConnected() node.SetDefaultACL() + node.SetDefaultFailover() } // GetRecordKey - get record key diff --git a/logic/server.go b/logic/server.go index 6e892bfb..764087fa 100644 --- a/logic/server.go +++ b/logic/server.go @@ -18,8 +18,12 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) +// EnterpriseCheckFuncs - can be set to run functions for EE var EnterpriseCheckFuncs []interface{} +// EnterpriseFailoverFunc - interface to control failover funcs +var EnterpriseFailoverFunc interface{} + // == Join, Checkin, and Leave for Server == // KUBERNETES_LISTEN_PORT - starting port for Kubernetes in order to use NodePort range diff --git a/models/node.go b/models/node.go index 25c01afc..b1819a99 100644 --- a/models/node.go +++ b/models/node.go @@ -104,6 +104,7 @@ type Node struct { // == PRO == DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"` OwnerID string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"` + Failover string `json:"failover" bson:"failover" yaml:"failover" validate:"checkyesorno"` } // NodesArray - used for node sorting @@ -297,6 +298,13 @@ func (node *Node) SetDefaultName() { } } +// Node.SetDefaultFailover - sets default value of failover status to no if not set +func (node *Node) SetDefaultFailover() { + if node.Failover == "" { + node.Failover = "yes" + } +} + // Node.Fill - fills other node data into calling node data if not set on calling node func (newNode *Node) Fill(currentNode *Node) { // TODO add new field for nftables present newNode.ID = currentNode.ID @@ -452,6 +460,10 @@ func (newNode *Node) Fill(currentNode *Node) { // TODO add new field for nftable newNode.DefaultACL = currentNode.DefaultACL } + if newNode.Failover == "" { + newNode.Failover = currentNode.Failover + } + newNode.TrafficKeys = currentNode.TrafficKeys } diff --git a/mq/handlers.go b/mq/handlers.go index 517010ca..78eff535 100644 --- a/mq/handlers.go +++ b/mq/handlers.go @@ -135,6 +135,28 @@ func UpdateMetrics(client mqtt.Client, msg mqtt.Message) { } } + if newMetrics.Connectivity != nil { + hasDisconnection := false + for k := range newMetrics.Connectivity { + if !newMetrics.Connectivity[k].Connected { + hasDisconnection = true + } + } + if hasDisconnection { + _, err := logic.EnterpriseFailoverFunc.(func(*models.Node) ([]models.Node, error))(¤tNode) + if err != nil { + logger.Log(0, "could failed to failover for node", currentNode.Name, "on network", currentNode.Network, "-", err.Error()) + } else { + if err := NodeUpdate(¤tNode); err != nil { + logger.Log(1, "error publishing node update to node", currentNode.Name, err.Error()) + } + if err := PublishPeerUpdate(¤tNode, true); err != nil { + logger.Log(1, "error publishing peer update after auto relay for node", currentNode.Name, err.Error()) + } + } + } + } + logger.Log(1, "updated node metrics", id, currentNode.Name) }() }