diff --git a/config/config.go b/config/config.go index fc1bcbf2..9160cc9f 100644 --- a/config/config.go +++ b/config/config.go @@ -70,6 +70,7 @@ type ServerConfig struct { DisplayKeys string `yaml:"displaykeys"` AzureTenant string `yaml:"azuretenant"` RCE string `yaml:"rce"` + Telemetry string `yaml:"telemetry"` } // SQLConfig - Generic SQL Config diff --git a/controllers/server_util.go b/controllers/server_util.go index c3e56e1d..84cd4566 100644 --- a/controllers/server_util.go +++ b/controllers/server_util.go @@ -4,9 +4,16 @@ import ( "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/servercfg" + "github.com/gravitl/netmaker/serverctl" ) func runServerPeerUpdate(network string, shouldPeerUpdate bool) error { + if servercfg.Telemetry() == "on" { + err := serverctl.TelemetryCheckpoint() + if err != nil { + logger.Log(1, "failed to send telemetry:", err.Error()) + } + } if servercfg.IsClientMode() != "on" { return nil } diff --git a/database/database.go b/database/database.go index c70ba111..b2769d1f 100644 --- a/database/database.go +++ b/database/database.go @@ -3,9 +3,12 @@ package database import ( "encoding/json" "errors" + "strings" "time" + "github.com/google/uuid" "github.com/gravitl/netmaker/logger" + "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" ) @@ -36,6 +39,12 @@ const PEERS_TABLE_NAME = "peers" // SERVERCONF_TABLE_NAME const SERVERCONF_TABLE_NAME = "serverconf" +// SERVER_UUID_TABLE_NAME +const SERVER_UUID_TABLE_NAME = "serveruuid" + +// SERVER_UUID_RECORD_KEY +const SERVER_UUID_RECORD_KEY = "serveruuid" + // DATABASE_FILENAME - database file name const DATABASE_FILENAME = "netmaker.db" @@ -105,7 +114,8 @@ func InitializeDatabase() error { time.Sleep(2 * time.Second) } createTables() - return nil + err := initializeUUID() + return err } func createTables() { @@ -118,6 +128,7 @@ func createTables() { createTable(INT_CLIENTS_TABLE_NAME) createTable(PEERS_TABLE_NAME) createTable(SERVERCONF_TABLE_NAME) + createTable(SERVER_UUID_TABLE_NAME) createTable(GENERATED_TABLE_NAME) } @@ -184,6 +195,25 @@ func FetchRecords(tableName string) (map[string]string, error) { return getCurrentDB()[FETCH_ALL].(func(string) (map[string]string, error))(tableName) } +// initializeUUID - create a UUID record for server if none exists +func initializeUUID() error { + records, err := FetchRecords(SERVER_UUID_TABLE_NAME) + if err != nil { + if !strings.Contains("could not find any records", err.Error()) { + return err + } + } else if len(records) > 0 { + return nil + } + telemetry := models.Telemetry{UUID: uuid.NewString()} + telJSON, err := json.Marshal(telemetry) + if err != nil { + return err + } + + return Insert(SERVER_UUID_RECORD_KEY, string(telJSON), SERVER_UUID_TABLE_NAME) +} + // CloseDB - closes a database gracefully func CloseDB() { getCurrentDB()[CLOSE_DB].(func())() diff --git a/go.mod b/go.mod index 70f37dc5..b63dd3cc 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,9 @@ require ( github.com/mdlayher/genetlink v1.0.0 // indirect github.com/mdlayher/netlink v1.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect google.golang.org/appengine v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 1843eb9d..98d74829 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0 h1:Y2hUrkfuM0on62KZOci/VLijlkdF/yeWU262BQgvcjE= +github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0/go.mod h1:oa2sAs9tGai3VldabTV0eWejt/O4/OOD7azP8GaikqU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -156,9 +158,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/txn2/txeh v1.3.0 h1:vnbv63htVMZCaQgLqVBxKvj2+HHHFUzNW7I183zjg3E= github.com/txn2/txeh v1.3.0/go.mod h1:O7M6gUTPeMF+vsa4c4Ipx3JDkOYrruB1Wry8QRsMcw8= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/logic/nodes.go b/logic/nodes.go index 073eeb20..4a286274 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -411,6 +411,8 @@ func SetNodeDefaults(node *models.Node) { node.SetDefaultMTU() node.SetDefaultIsRelayed() node.SetDefaultIsRelay() + node.SetDefaultIsDocker() + node.SetDefaultIsK8S() node.KeyUpdateTimeStamp = time.Now().Unix() } diff --git a/main.go b/main.go index 4e3b71ca..eba2c60f 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,10 @@ func initialize() { // Client Mode Prereq Check } logger.Log(0, "database successfully connected") + err = serverctl.TelemetryCheckpoint() + if err != nil { + logger.Log(1, "Failed to send telemetry: ", err.Error()) + } var authProvider = auth.InitializeAuthProvider() if authProvider != "" { logger.Log(0, "OAuth provider,", authProvider+",", "initialized") diff --git a/models/node.go b/models/node.go index 16f65afd..3cd629b2 100644 --- a/models/node.go +++ b/models/node.go @@ -54,6 +54,8 @@ type Node struct { IsRelayed string `json:"isrelayed" bson:"isrelayed" yaml:"isrelayed"` IsPending string `json:"ispending" bson:"ispending" yaml:"ispending"` IsRelay string `json:"isrelay" bson:"isrelay" yaml:"isrelay" validate:"checkyesorno"` + IsDocker string `json:"isdocker" bson:"isdocker" yaml:"isdocker" validate:"checkyesorno"` + IsK8S string `json:"isk8s" bson:"isk8s" yaml:"isk8s" validate:"checkyesorno"` IsEgressGateway string `json:"isegressgateway" bson:"isegressgateway" yaml:"isegressgateway"` IsIngressGateway string `json:"isingressgateway" bson:"isingressgateway" yaml:"isingressgateway"` EgressGatewayRanges []string `json:"egressgatewayranges" bson:"egressgatewayranges" yaml:"egressgatewayranges"` @@ -122,6 +124,20 @@ func (node *Node) SetDefaultIsRelay() { } } +// Node.SetDefaultIsDocker - set default isdocker +func (node *Node) SetDefaultIsDocker() { + if node.IsDocker == "" { + node.IsDocker = "no" + } +} + +// Node.SetDefaultIsK8S - set default isk8s +func (node *Node) SetDefaultIsK8S() { + if node.IsK8S == "" { + node.IsK8S = "no" + } +} + // Node.SetDefaultEgressGateway - sets default egress gateway status func (node *Node) SetDefaultEgressGateway() { if node.IsEgressGateway == "" { @@ -381,6 +397,12 @@ func (newNode *Node) Fill(currentNode *Node) { if newNode.IsRelayed == "" { newNode.IsRelayed = currentNode.IsRelayed } + if newNode.IsDocker == "" { + newNode.IsDocker = currentNode.IsDocker + } + if newNode.IsK8S == "" { + newNode.IsK8S = currentNode.IsK8S + } if newNode.Version == "" { newNode.Version = currentNode.Version } diff --git a/models/structs.go b/models/structs.go index 16b2ae77..3e772bc3 100644 --- a/models/structs.go +++ b/models/structs.go @@ -163,3 +163,9 @@ type ServerUpdateData struct { UpdatePeers bool `json:"updatepeers" bson:"updatepeers"` Node Node `json:"servernode" bson:"servernode"` } + +// Telemetry - contains UUID of the server and timestamp of last send to posthog +type Telemetry struct { + UUID string `json:"uuid" bson:"uuid"` + LastSend int64 `json:"lastsend" bson:"lastsend"` +} diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 224d1862..08c90a1c 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -85,6 +85,7 @@ func GetServerConfig() config.ServerConfig { } else { cfg.RCE = "off" } + cfg.Telemetry = Telemetry() return cfg } @@ -319,6 +320,18 @@ func IsClientMode() string { return isclient } +// Telemetry - checks if telemetry data should be sent +func Telemetry() string { + telemetry := "on" + if os.Getenv("TELEMETRY") == "off" { + telemetry = "off" + } + if config.Config.Server.Telemetry == "off" { + telemetry = "off" + } + return telemetry +} + // IsDNSMode - should it run with DNS func IsDNSMode() bool { isdns := true diff --git a/serverctl/telemetry.go b/serverctl/telemetry.go new file mode 100644 index 00000000..fd64492c --- /dev/null +++ b/serverctl/telemetry.go @@ -0,0 +1,181 @@ +package serverctl + +import ( + "encoding/json" + "time" + + "github.com/gravitl/netmaker/database" + "github.com/gravitl/netmaker/logger" + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/models" + "github.com/gravitl/netmaker/servercfg" + "github.com/posthog/posthog-go" +) + +// POSTHOG_PUB_KEY - Key for sending data to PostHog +const POSTHOG_PUB_KEY = "phc_1vEXhPOA1P7HP5jP2dVU9xDTUqXHAelmtravyZ1vvES" + +// POSTHOG_ENDPOINT - Endpoint of PostHog server +const POSTHOG_ENDPOINT = "https://app.posthog.com" + +// TELEMETRY_HOURS_BETWEEN_SEND - How long to wait before sending telemetry to server (24 hours) +const TELEMETRY_HOURS_BETWEEN_SEND = 24 + +// TelemetryCheckpoint - Checks if 24 hours has passed since telemetry was last sent. If so, sends telemetry data to posthog +func TelemetryCheckpoint() error { + + // if telemetry is turned off, return without doing anything + if servercfg.Telemetry() == "off" { + return nil + } + // get the telemetry record in the DB, which contains a timestamp + telRecord, err := fetchTelemetryRecord() + if err != nil { + return err + } + sendtime := time.Unix(telRecord.LastSend, 0).Add(time.Hour * time.Duration(TELEMETRY_HOURS_BETWEEN_SEND)) + // can set to 2 minutes for testing + //sendtime := time.Unix(telRecord.LastSend, 0).Add(time.Minute * 2) + enoughTimeElapsed := time.Now().After(sendtime) + // if more than 24 hours has elapsed, send telemetry to posthog + if enoughTimeElapsed { + err = sendTelemetry(telRecord.UUID) + if err != nil { + logger.Log(1, err.Error()) + } + } + return nil +} + +// sendTelemetry - gathers telemetry data and sends to posthog +func sendTelemetry(serverUUID string) error { + // get telemetry data + d, err := fetchTelemetryData() + if err != nil { + return err + } + client, err := posthog.NewWithConfig(POSTHOG_PUB_KEY, posthog.Config{Endpoint: POSTHOG_ENDPOINT}) + if err != nil { + return err + } + defer client.Close() + + // send to posthog + err = client.Enqueue(posthog.Capture{ + DistinctId: serverUUID, + Event: "daily checkin", + Properties: posthog.NewProperties(). + Set("nodes", d.Nodes). + Set("non-server nodes", d.Count.NonServer). + Set("extclients", d.ExtClients). + Set("users", d.Users). + Set("networks", d.Networks). + Set("linux", d.Count.Linux). + Set("darwin", d.Count.MacOS). + Set("windows", d.Count.Windows). + Set("freebsd", d.Count.FreeBSD). + Set("docker", d.Count.Docker). + Set("k8s", d.Count.K8S). + Set("version", d.Version), + }) + if err != nil { + return err + } + //set telemetry timestamp for server, restarts 24 hour cycle + return setTelemetryTimestamp(serverUUID) +} + +// fetchTelemetry - fetches telemetry data: count of various object types in DB +func fetchTelemetryData() (telemetryData, error) { + var data telemetryData + + data.ExtClients = getDBLength(database.EXT_CLIENT_TABLE_NAME) + data.Users = getDBLength(database.USERS_TABLE_NAME) + data.Networks = getDBLength(database.NETWORKS_TABLE_NAME) + data.Version = servercfg.GetVersion() + nodes, err := logic.GetAllNodes() + if err == nil { + data.Nodes = len(nodes) + data.Count = getClientCount(nodes) + } + return data, err +} + +// setTelemetryTimestamp - Give the entry in the DB a new timestamp +func setTelemetryTimestamp(uuid string) error { + lastsend := time.Now().Unix() + var serverTelData = models.Telemetry{ + UUID: uuid, + LastSend: lastsend, + } + jsonObj, err := json.Marshal(serverTelData) + if err != nil { + return err + } + err = database.Insert(database.SERVER_UUID_RECORD_KEY, string(jsonObj), database.SERVER_UUID_TABLE_NAME) + return err +} + +// getClientCount - returns counts of nodes with various OS types and conditions +func getClientCount(nodes []models.Node) clientCount { + var count clientCount + for _, node := range nodes { + switch node.OS { + case "macos": + count.MacOS += 1 + case "windows": + count.Windows += 1 + case "linux": + count.Linux += 1 + case "freebsd": + count.FreeBSD += 1 + } + if !(node.IsServer == "yes") { + count.NonServer += 1 + } + } + return count +} + +// fetchTelemetryRecord - get the existing UUID and Timestamp from the DB +func fetchTelemetryRecord() (models.Telemetry, error) { + var rawData string + var telObj models.Telemetry + var err error + rawData, err = database.FetchRecord(database.SERVER_UUID_TABLE_NAME, database.SERVER_UUID_RECORD_KEY) + if err != nil { + return telObj, err + } + err = json.Unmarshal([]byte(rawData), &telObj) + return telObj, err +} + +// getDBLength - get length of DB to get count of objects +func getDBLength(dbname string) int { + data, err := database.FetchRecords(dbname) + if err != nil { + return 0 + } + return len(data) +} + +// telemetryData - What data to send to posthog +type telemetryData struct { + Nodes int + ExtClients int + Users int + Count clientCount + Networks int + Version string +} + +// clientCount - What types of netclients we're tallying +type clientCount struct { + MacOS int + Windows int + Linux int + FreeBSD int + K8S int + Docker int + NonServer int +}