netmaker/logic/zombie.go

162 lines
4.8 KiB
Go
Raw Normal View History

2022-06-14 03:19:25 +08:00
package logic
import (
"context"
"time"
"github.com/google/uuid"
2022-06-14 03:19:25 +08:00
"github.com/gravitl/netmaker/logger"
"github.com/gravitl/netmaker/models"
)
const (
// ZOMBIE_TIMEOUT - timeout in hours for checking zombie status
ZOMBIE_TIMEOUT = 6
2022-06-14 20:27:30 +08:00
// ZOMBIE_DELETE_TIME - timeout in minutes for zombie node deletion
ZOMBIE_DELETE_TIME = 10
2022-06-14 03:19:25 +08:00
)
var (
2023-02-14 20:01:38 +08:00
zombies []uuid.UUID
hostZombies []uuid.UUID
newZombie chan uuid.UUID = make(chan (uuid.UUID), 10)
newHostZombie chan uuid.UUID = make(chan (uuid.UUID), 10)
2022-06-14 03:19:25 +08:00
)
2023-02-14 03:52:39 +08:00
// CheckZombies - checks if new node has same hostid as existing node
2022-06-14 03:19:25 +08:00
// if so, existing node is added to zombie node quarantine list
// also cleans up nodes past their expiration date
2023-02-14 03:52:39 +08:00
func CheckZombies(newnode *models.Node) {
2022-06-14 03:19:25 +08:00
nodes, err := GetNetworkNodes(newnode.Network)
if err != nil {
logger.Log(1, "Failed to retrieve network nodes", newnode.Network, err.Error())
return
}
for _, node := range nodes {
2023-02-03 00:16:54 +08:00
if node.ID == newnode.ID {
//skip self
continue
}
2023-02-03 00:16:54 +08:00
if node.HostID == newnode.HostID || time.Now().After(node.ExpirationDateTime) {
logger.Log(0, "adding ", node.ID.String(), " to zombie list")
2023-02-14 20:01:38 +08:00
newZombie <- node.ID
2023-02-14 03:52:39 +08:00
}
}
}
// checkForZombieHosts - checks if new host has the same macAddress as an existing host
// if true, existing host is added to host zombie collection
func checkForZombieHosts(h *models.Host) {
hosts, err := GetAllHosts()
if err != nil {
logger.Log(3, "errror retrieving all hosts", err.Error())
}
for _, existing := range hosts {
if existing.ID == h.ID {
//probably an unnecessary check as new host should not be in database yet, but just in case
//skip self
continue
}
if existing.MacAddress.String() == h.MacAddress.String() {
//add to hostZombies
2023-02-14 20:01:38 +08:00
newHostZombie <- existing.ID
2023-02-14 03:52:39 +08:00
//add all nodes belonging to host to zombile list
for _, node := range existing.Nodes {
id, err := uuid.Parse(node)
if err != nil {
logger.Log(3, "error parsing uuid from host.Nodes", err.Error())
continue
}
2023-02-14 20:01:38 +08:00
newHostZombie <- id
2023-02-14 03:52:39 +08:00
}
2022-06-14 03:19:25 +08:00
}
}
}
// ManageZombies - goroutine which adds/removes/deletes nodes from the zombie node quarantine list
2023-01-27 21:27:54 +08:00
func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) {
2022-10-04 04:27:09 +08:00
logger.Log(2, "Zombie management started")
InitializeZombies()
2022-06-14 03:19:25 +08:00
for {
select {
case <-ctx.Done():
2023-02-14 20:58:54 +08:00
close(peerUpdate)
2022-06-14 03:19:25 +08:00
return
2023-02-14 20:01:38 +08:00
case id := <-newZombie:
zombies = append(zombies, id)
case id := <-newHostZombie:
hostZombies = append(hostZombies, id)
case <-time.After(time.Hour * ZOMBIE_TIMEOUT): // run this check 4 times a day
logger.Log(3, "checking for zombie nodes")
2022-08-24 06:48:05 +08:00
if len(zombies) > 0 {
2022-08-24 04:35:36 +08:00
for i := len(zombies) - 1; i >= 0; i-- {
node, err := GetNodeByID(zombies[i].String())
2022-08-24 06:48:05 +08:00
if err != nil {
logger.Log(1, "error retrieving zombie node", zombies[i].String(), err.Error())
logger.Log(1, "deleting ", node.ID.String(), " from zombie list")
2022-10-04 04:27:09 +08:00
zombies = append(zombies[:i], zombies[i+1:]...)
2022-06-14 20:27:30 +08:00
continue
2022-06-14 03:19:25 +08:00
}
2023-01-27 21:27:54 +08:00
if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME || time.Now().After(node.ExpirationDateTime) {
2022-11-15 22:52:17 +08:00
if err := DeleteNode(&node, true); err != nil {
logger.Log(1, "error deleting zombie node", zombies[i].String(), err.Error())
2022-08-24 06:48:05 +08:00
continue
}
2023-01-27 21:27:54 +08:00
node.Action = models.NODE_DELETE
peerUpdate <- &node
logger.Log(1, "deleting zombie node", node.ID.String())
2022-08-24 06:48:05 +08:00
zombies = append(zombies[:i], zombies[i+1:]...)
}
2022-06-14 03:19:25 +08:00
}
}
2023-02-14 03:52:39 +08:00
if len(hostZombies) > 0 {
logger.Log(3, "checking host zombies")
for i := len(hostZombies) - 1; i >= 0; i-- {
host, err := GetHost(hostZombies[i].String())
if err != nil {
logger.Log(1, "error retrieving zombie host", err.Error())
logger.Log(1, "deleting ", host.ID.String(), " from zombie list")
zombies = append(zombies[:i], zombies[i+1:]...)
continue
}
if len(host.Nodes) == 0 {
if err := RemoveHost(host); err != nil {
logger.Log(0, "error deleting zombie host", host.ID.String(), err.Error())
}
}
}
}
2022-06-14 03:19:25 +08:00
}
}
}
2022-06-14 20:27:30 +08:00
// InitializeZombies - populates the zombie quarantine list (should be called from initialization)
2022-10-04 04:27:09 +08:00
func InitializeZombies() {
2022-06-14 03:19:25 +08:00
nodes, err := GetAllNodes()
if err != nil {
logger.Log(1, "failed to retrieve nodes", err.Error())
return
}
for _, node := range nodes {
othernodes, err := GetNetworkNodes(node.Network)
if err != nil {
logger.Log(1, "failled to retrieve nodes for network", node.Network, err.Error())
continue
}
for _, othernode := range othernodes {
2022-06-14 03:27:49 +08:00
if node.ID == othernode.ID {
continue
}
if node.HostID == othernode.HostID {
if node.LastCheckIn.After(othernode.LastCheckIn) {
2023-02-14 20:01:38 +08:00
newZombie <- othernode.ID
logger.Log(1, "adding", othernode.ID.String(), "to zombie list")
2022-06-14 03:19:25 +08:00
} else {
2023-02-14 20:01:38 +08:00
newZombie <- node.ID
logger.Log(1, "adding", node.ID.String(), "to zombie list")
2022-06-14 03:19:25 +08:00
}
}
}
}
}