From 230e062c844758de3e7e7c3326e23e009c1aaba7 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 28 Jun 2023 20:33:06 +0530 Subject: [PATCH] GRA-1298: License check changes, free tier limits for saas (#2418) * set free tier limits through config * add host limit to config * check for host limit on free tier * fix license validation, replace node limit with hosts * add hosts to telemetry data * debug init * validate license every 1hr * hook manager, api to fetch server usage * hook manager, server usage api * encode json server usage api * update ngork url * update license validation endpoint * avoid setting limits on eer * adding hotfix * correct users limits env var * add comments to exported funcs --------- Co-authored-by: afeiszli --- config/config.go | 5 ++++ controllers/limits.go | 10 +------ controllers/server.go | 38 +++++++++++++++++++++++++++ ee/initialize.go | 21 +++------------ ee/license.go | 32 +++++++++-------------- ee/types.go | 58 ++++++++++++++--------------------------- ee/util.go | 9 ++++--- logic/hosts.go | 9 ++++++- logic/serverconf.go | 13 +++++++-- logic/telemetry.go | 3 +++ logic/timer.go | 36 +++++++++++++++++++++++++ main.go | 7 ++++- models/structs.go | 16 ++++++++++++ mq/emqx.go | 7 ++++- servercfg/serverconf.go | 53 +++++++++++++++++++++++++++++++++++++ 15 files changed, 225 insertions(+), 92 deletions(-) diff --git a/config/config.go b/config/config.go index fe95fea9..0dbafc85 100644 --- a/config/config.go +++ b/config/config.go @@ -83,6 +83,11 @@ type ServerConfig struct { TurnUserName string `yaml:"turn_username"` TurnPassword string `yaml:"turn_password"` UseTurn bool `yaml:"use_turn"` + UsersLimit int `yaml:"user_limit"` + ClientsLimit int `yaml:"client_limit"` + NetworksLimit int `yaml:"network_limit"` + HostsLimit int `yaml:"host_limit"` + DeployedByOperator bool `yaml:"deployed_by_operator"` } // ProxyMode - default proxy mode for server diff --git a/controllers/limits.go b/controllers/limits.go index 268ada2d..2b4cee15 100644 --- a/controllers/limits.go +++ b/controllers/limits.go @@ -6,7 +6,6 @@ import ( "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" - "github.com/gravitl/netmaker/servercfg" ) // limit consts @@ -23,20 +22,13 @@ func checkFreeTierLimits(limit_choice int, next http.Handler) http.HandlerFunc { Code: http.StatusForbidden, Message: "free tier limits exceeded on networks", } - if logic.Free_Tier && servercfg.Is_EE { // check that free tier limits not exceeded + if logic.Free_Tier { // check that free tier limits not exceeded if limit_choice == networks_l { currentNetworks, err := logic.GetNetworks() if (err != nil && !database.IsEmptyRecord(err)) || len(currentNetworks) >= logic.Networks_Limit { logic.ReturnErrorResponse(w, r, errorResponse) return } - } else if limit_choice == node_l { - nodes, err := logic.GetAllNodes() - if (err != nil && !database.IsEmptyRecord(err)) || len(nodes) >= logic.Node_Limit { - errorResponse.Message = "free tier limits exceeded on nodes" - logic.ReturnErrorResponse(w, r, errorResponse) - return - } } else if limit_choice == users_l { users, err := logic.GetUsers() if (err != nil && !database.IsEmptyRecord(err)) || len(users) >= logic.Users_Limit { diff --git a/controllers/server.go b/controllers/server.go index bcbf7c9a..e9aa1b76 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -22,6 +22,38 @@ func serverHandlers(r *mux.Router) { r.HandleFunc("/api/server/getconfig", allowUsers(http.HandlerFunc(getConfig))).Methods(http.MethodGet) r.HandleFunc("/api/server/getserverinfo", Authorize(true, false, "node", http.HandlerFunc(getServerInfo))).Methods(http.MethodGet) r.HandleFunc("/api/server/status", http.HandlerFunc(getStatus)).Methods(http.MethodGet) + r.HandleFunc("/api/server/usage", Authorize(true, false, "user", http.HandlerFunc(getUsage))).Methods(http.MethodGet) +} +func getUsage(w http.ResponseWriter, r *http.Request) { + type usage struct { + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` + Users int `json:"users"` + } + var serverUsage usage + hosts, err := logic.GetAllHosts() + if err == nil { + serverUsage.Hosts = len(hosts) + } + clients, err := logic.GetAllExtClients() + if err == nil { + serverUsage.Clients = len(clients) + } + users, err := logic.GetUsers() + if err == nil { + serverUsage.Users = len(users) + } + networks, err := logic.GetNetworks() + if err == nil { + serverUsage.Networks = len(networks) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(models.SuccessResponse{ + Code: http.StatusOK, + Response: serverUsage, + }) + } // swagger:route GET /api/server/status server getStatus @@ -41,6 +73,12 @@ func getStatus(w http.ResponseWriter, r *http.Request) { type status struct { DB bool `json:"db_connected"` Broker bool `json:"broker_connected"` + Usage struct { + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` + Users int `json:"users"` + } `json:"usage"` } currentServerStatus := status{ diff --git a/ee/initialize.go b/ee/initialize.go index ab5e3ba0..455ee59d 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -16,6 +16,7 @@ import ( // InitEE - Initialize EE Logic func InitEE() { setIsEnterprise() + servercfg.Is_EE = true models.SetLogo(retrieveEELogo()) controller.HttpHandlers = append( controller.HttpHandlers, @@ -27,13 +28,8 @@ func InitEE() { logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() { // == License Handling == ValidateLicense() - if Limits.FreeTier { - logger.Log(0, "proceeding with Free Tier license") - logic.SetFreeTierForTelemetry(true) - } else { - logger.Log(0, "proceeding with Paid Tier license") - logic.SetFreeTierForTelemetry(false) - } + logger.Log(0, "proceeding with Paid Tier license") + logic.SetFreeTierForTelemetry(false) // == End License Handling == AddLicenseHooks() resetFailover() @@ -46,17 +42,6 @@ func InitEE() { logic.AllowClientNodeAccess = eelogic.RemoveDeniedNodeFromClient } -func setControllerLimits() { - logic.Node_Limit = Limits.Nodes - logic.Users_Limit = Limits.Users - logic.Clients_Limit = Limits.Clients - logic.Free_Tier = Limits.FreeTier - servercfg.Is_EE = true - if logic.Free_Tier { - logic.Networks_Limit = 3 - } -} - func resetFailover() { nets, err := logic.GetNetworks() if err == nil { diff --git a/ee/license.go b/ee/license.go index 521326ec..d9b197ab 100644 --- a/ee/license.go +++ b/ee/license.go @@ -9,12 +9,13 @@ import ( "encoding/json" "fmt" "io" - "math" "net/http" + "time" "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/netclient/ncutils" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/nacl/box" @@ -31,8 +32,14 @@ type apiServerConf struct { // AddLicenseHooks - adds the validation and cache clear hooks func AddLicenseHooks() { - logic.AddHook(ValidateLicense) - logic.AddHook(ClearLicenseCache) + logic.HookManagerCh <- models.HookDetails{ + Hook: ValidateLicense, + Interval: time.Hour, + } + logic.HookManagerCh <- models.HookDetails{ + Hook: ClearLicenseCache, + Interval: time.Hour, + } } // ValidateLicense - the initial license check for netmaker server @@ -58,8 +65,8 @@ func ValidateLicense() error { } licenseSecret := LicenseSecret{ - UserID: netmakerAccountID, - Limits: getCurrentServerLimit(), + AssociatedID: netmakerAccountID, + Limits: getCurrentServerLimit(), } secretData, err := json.Marshal(&licenseSecret) @@ -92,17 +99,6 @@ func ValidateLicense() error { logger.FatalLog0(errValidation.Error()) } - Limits.Networks = math.MaxInt - Limits.FreeTier = license.FreeTier == "yes" - Limits.Clients = license.LimitClients - Limits.Nodes = license.LimitNodes - Limits.Servers = license.LimitServers - Limits.Users = license.LimitUsers - if Limits.FreeTier { - Limits.Networks = 3 - } - setControllerLimits() - logger.Log(0, "License validation succeeded!") return nil } @@ -167,6 +163,7 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro } msg := ValidateLicenseRequest{ + LicenseKey: servercfg.GetLicenseKey(), NmServerPubKey: base64encode(publicKeyBytes), EncryptedPart: base64encode(encryptedData), } @@ -180,9 +177,6 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro if err != nil { return nil, err } - reqParams := req.URL.Query() - reqParams.Add("licensevalue", servercfg.GetLicenseKey()) - req.URL.RawQuery = reqParams.Encode() req.Header.Add("Content-Type", "application/json") req.Header.Add("Accept", "application/json") client := &http.Client{} diff --git a/ee/types.go b/ee/types.go index f59d21fd..17b5dd33 100644 --- a/ee/types.go +++ b/ee/types.go @@ -3,7 +3,7 @@ package ee import "fmt" const ( - api_endpoint = "https://api.controller.netmaker.io/api/v1/license/validate" + api_endpoint = "https://api.accounts.netmaker.io/api/v1/license/validate" license_cache_key = "license_response_cache" license_validation_err_msg = "invalid license" server_id_key = "nm-server-id" @@ -11,38 +11,17 @@ const ( var errValidation = fmt.Errorf(license_validation_err_msg) -// Limits - limits to be referenced throughout server -var Limits = GlobalLimits{ - Servers: 0, - Users: 0, - Nodes: 0, - Clients: 0, - Networks: 0, - FreeTier: false, -} - -// GlobalLimits - struct for holding global limits on this netmaker server in memory -type GlobalLimits struct { - Servers int - Users int - Nodes int - Clients int - FreeTier bool - Networks int -} - // LicenseKey - the license key struct representation with associated data type LicenseKey struct { - LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key - Expiration int64 `json:"expiration"` - LimitServers int `json:"limit_servers"` - LimitUsers int `json:"limit_users"` - LimitNodes int `json:"limit_nodes"` - LimitClients int `json:"limit_clients"` - Metadata string `json:"metadata"` - SubscriptionID string `json:"subscription_id"` // for a paid subscription (non-free-tier license) - FreeTier string `json:"free_tier"` // yes if free tier - IsActive string `json:"is_active"` // yes if active + LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key + Expiration int64 `json:"expiration"` + LimitServers int `json:"limit_servers"` + LimitUsers int `json:"limit_users"` + LimitHosts int `json:"limit_hosts"` + LimitNetworks int `json:"limit_networks"` + LimitClients int `json:"limit_clients"` + Metadata string `json:"metadata"` + IsActive bool `json:"is_active"` // yes if active } // ValidatedLicense - the validated license struct @@ -53,28 +32,31 @@ type ValidatedLicense struct { // LicenseSecret - the encrypted struct for sending user-id type LicenseSecret struct { - UserID string `json:"user_id" binding:"required"` // UUID for user foreign key to User table - Limits LicenseLimits `json:"limits" binding:"required"` + AssociatedID string `json:"associated_id" binding:"required"` // UUID for user foreign key to User table + Limits LicenseLimits `json:"limits" binding:"required"` } // LicenseLimits - struct license limits type LicenseLimits struct { - Servers int `json:"servers" binding:"required"` - Users int `json:"users" binding:"required"` - Nodes int `json:"nodes" binding:"required"` - Clients int `json:"clients" binding:"required"` + Servers int `json:"servers"` + Users int `json:"users"` + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` } // LicenseLimits.SetDefaults - sets the default values for limits func (l *LicenseLimits) SetDefaults() { l.Clients = 0 l.Servers = 1 - l.Nodes = 0 + l.Hosts = 0 l.Users = 1 + l.Networks = 0 } // ValidateLicenseRequest - used for request to validate license endpoint type ValidateLicenseRequest struct { + LicenseKey string `json:"license_key" binding:"required"` NmServerPubKey string `json:"nm_server_pub_key" binding:"required"` // Netmaker server public key used to send data back to Netmaker for the Netmaker server to decrypt (eg output from validating license) EncryptedPart string `json:"secret" binding:"required"` } diff --git a/ee/util.go b/ee/util.go index a45bc1a4..b63cf158 100644 --- a/ee/util.go +++ b/ee/util.go @@ -30,12 +30,11 @@ func base64decode(input string) []byte { return bytes } - func getCurrentServerLimit() (limits LicenseLimits) { limits.SetDefaults() - nodes, err := logic.GetAllNodes() + hosts, err := logic.GetAllHosts() if err == nil { - limits.Nodes = len(nodes) + limits.Hosts = len(hosts) } clients, err := logic.GetAllExtClients() if err == nil { @@ -45,5 +44,9 @@ func getCurrentServerLimit() (limits LicenseLimits) { if err == nil { limits.Users = len(users) } + networks, err := logic.GetNetworks() + if err == nil { + limits.Networks = len(networks) + } return } diff --git a/logic/hosts.go b/logic/hosts.go index 34fa9766..9f6db42a 100644 --- a/logic/hosts.go +++ b/logic/hosts.go @@ -93,7 +93,14 @@ func GetHost(hostid string) (*models.Host, error) { // CreateHost - creates a host if not exist func CreateHost(h *models.Host) error { - _, err := GetHost(h.ID.String()) + hosts, err := GetAllHosts() + if err != nil && !database.IsEmptyRecord(err) { + return err + } + if len(hosts) >= Hosts_Limit { + return errors.New("free tier limits exceeded on hosts") + } + _, err = GetHost(h.ID.String()) if (err != nil && !database.IsEmptyRecord(err)) || (err == nil) { return ErrHostExists } diff --git a/logic/serverconf.go b/logic/serverconf.go index 01c39346..4d73980d 100644 --- a/logic/serverconf.go +++ b/logic/serverconf.go @@ -4,17 +4,18 @@ import ( "encoding/json" "github.com/gravitl/netmaker/database" + "github.com/gravitl/netmaker/servercfg" ) var ( - // Node_Limit - dummy var for community - Node_Limit = 1000000000 // Networks_Limit - dummy var for community Networks_Limit = 1000000000 // Users_Limit - dummy var for community Users_Limit = 1000000000 // Clients_Limit - dummy var for community Clients_Limit = 1000000000 + // Hosts_Limit - dummy var for community + Hosts_Limit = 1000000000 // Free_Tier - specifies if free tier Free_Tier = false ) @@ -85,3 +86,11 @@ func StoreJWTSecret(privateKey string) error { } return database.Insert("nm-jwt-secret", string(data), database.SERVERCONF_TABLE_NAME) } + +func SetFreeTierLimits() { + Free_Tier = true + Users_Limit = servercfg.GetUserLimit() + Clients_Limit = servercfg.GetClientLimit() + Networks_Limit = servercfg.GetNetworkLimit() + Hosts_Limit = servercfg.GetHostLimit() +} diff --git a/logic/telemetry.go b/logic/telemetry.go index 4b9e8635..e867cbc8 100644 --- a/logic/telemetry.go +++ b/logic/telemetry.go @@ -60,6 +60,7 @@ func sendTelemetry() error { Event: "daily checkin", Properties: posthog.NewProperties(). Set("nodes", d.Nodes). + Set("hosts", d.Hosts). Set("servers", d.Servers). Set("non-server nodes", d.Count.NonServer). Set("extclients", d.ExtClients). @@ -84,6 +85,7 @@ func fetchTelemetryData() (telemetryData, error) { data.ExtClients = getDBLength(database.EXT_CLIENT_TABLE_NAME) data.Users = getDBLength(database.USERS_TABLE_NAME) data.Networks = getDBLength(database.NETWORKS_TABLE_NAME) + data.Hosts = getDBLength(database.HOSTS_TABLE_NAME) data.Version = servercfg.GetVersion() data.Servers = getServerCount() nodes, err := GetAllNodes() @@ -167,6 +169,7 @@ func getDBLength(dbname string) int { // telemetryData - What data to send to posthog type telemetryData struct { Nodes int + Hosts int ExtClients int Users int Count clientCount diff --git a/logic/timer.go b/logic/timer.go index 0ea8d1db..89070f2f 100644 --- a/logic/timer.go +++ b/logic/timer.go @@ -1,10 +1,13 @@ package logic import ( + "context" "fmt" + "sync" "time" "github.com/gravitl/netmaker/logger" + "github.com/gravitl/netmaker/models" ) // == Constants == @@ -12,6 +15,9 @@ import ( // How long to wait before sending telemetry to server (24 hours) const timer_hours_between_runs = 24 +// HookManagerCh - channel to add any new hooks +var HookManagerCh = make(chan models.HookDetails, 2) + // == Public == // TimerCheckpoint - Checks if 24 hours has passed since telemetry was last sent. If so, sends telemetry data to posthog @@ -40,6 +46,36 @@ func AddHook(ifaceToAdd interface{}) { timeHooks = append(timeHooks, ifaceToAdd) } +// StartHookManager - listens on `HookManagerCh` to run any hook +func StartHookManager(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + logger.Log(0, "## Stopping Hook Manager") + return + case newhook := <-HookManagerCh: + wg.Add(1) + go addHookWithInterval(ctx, wg, newhook.Hook, newhook.Interval) + } + } +} + +func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() error, interval time.Duration) { + defer wg.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + hook() + } + } + +} + // == private == // timeHooks - functions to run once a day, functions must take no parameters diff --git a/main.go b/main.go index 5110b841..5c5580bd 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,9 @@ func main() { initialize() // initial db and acls setGarbageCollection() setVerbosity() + if servercfg.DeployedByOperator() && !servercfg.Is_EE { + logic.SetFreeTierLimits() + } defer database.CloseDB() ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) defer stop() @@ -89,7 +92,6 @@ func initialize() { // Client Mode Prereq Check if err != nil { logger.Log(1, "Timer error occurred: ", err.Error()) } - logic.EnterpriseCheck() var authProvider = auth.InitializeAuthProvider() @@ -150,6 +152,9 @@ func startControllers(wg *sync.WaitGroup, ctx context.Context) { // starts the stun server wg.Add(1) go stunserver.Start(wg, ctx) + + wg.Add(1) + go logic.StartHookManager(ctx, wg) } // Should we be using a context vice a waitgroup???????????? diff --git a/models/structs.go b/models/structs.go index 2343cf6c..bff547c3 100644 --- a/models/structs.go +++ b/models/structs.go @@ -2,6 +2,7 @@ package models import ( "strings" + "time" jwt "github.com/golang-jwt/jwt/v4" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -274,3 +275,18 @@ type StunServer struct { Domain string `json:"domain" yaml:"domain"` Port int `json:"port" yaml:"port"` } + +// HookDetails - struct to hold hook info +type HookDetails struct { + Hook func() error + Interval time.Duration +} + +// LicenseLimits - struct license limits +type LicenseLimits struct { + Servers int `json:"servers"` + Users int `json:"users"` + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` +} diff --git a/mq/emqx.go b/mq/emqx.go index cd6eb2d6..27776fe3 100644 --- a/mq/emqx.go +++ b/mq/emqx.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "github.com/gravitl/netmaker/servercfg" ) +const already_exists = "ALREADY_EXISTS" + type ( emqxUser struct { UserID string `json:"user_id"` @@ -99,7 +102,9 @@ func CreateEmqxUser(username, password string, admin bool) error { if err != nil { return err } - return fmt.Errorf("error creating EMQX user %v", string(msg)) + if !strings.Contains(string(msg), already_exists) { + return fmt.Errorf("error creating EMQX user %v", string(msg)) + } } return nil } diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index ec8b6ef0..185fd870 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gravitl/netmaker/config" + "github.com/gravitl/netmaker/models" ) @@ -741,6 +742,58 @@ func IsProxyEnabled() bool { return enabled } +// GetNetworkLimit - fetches free tier limits on users +func GetUserLimit() int { + var userslimit int + if os.Getenv("USERS_LIMIT") != "" { + userslimit, _ = strconv.Atoi(os.Getenv("USERS_LIMIT")) + } else { + userslimit = config.Config.Server.UsersLimit + } + return userslimit +} + +// GetNetworkLimit - fetches free tier limits on networks +func GetNetworkLimit() int { + var networkslimit int + if os.Getenv("NETWORKS_LIMIT") != "" { + networkslimit, _ = strconv.Atoi(os.Getenv("NETWORKS_LIMIT")) + } else { + networkslimit = config.Config.Server.NetworksLimit + } + return networkslimit +} + +// GetClientLimit - fetches free tier limits on ext. clients +func GetClientLimit() int { + var clientsLimit int + if os.Getenv("CLIENTS_LIMIT") != "" { + clientsLimit, _ = strconv.Atoi(os.Getenv("CLIENTS_LIMIT")) + } else { + clientsLimit = config.Config.Server.ClientsLimit + } + return clientsLimit +} + +// GetHostLimit - fetches free tier limits on hosts +func GetHostLimit() int { + var hostsLimit int + if os.Getenv("HOSTS_LIMIT") != "" { + hostsLimit, _ = strconv.Atoi(os.Getenv("HOSTS_LIMIT")) + } else { + hostsLimit = config.Config.Server.HostsLimit + } + return hostsLimit +} + +// DeployedByOperator - returns true if the instance is deployed by netmaker operator +func DeployedByOperator() bool { + if os.Getenv("DEPLOYED_BY_OPERATOR") != "" { + return os.Getenv("DEPLOYED_BY_OPERATOR") == "true" + } + return config.Config.Server.DeployedByOperator +} + // GetDefaultProxyMode - default proxy mode for a server func GetDefaultProxyMode() config.ProxyMode { var (