From d2adf88b21b6bd26f179660172437cf67afc297c Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 20 Feb 2023 15:56:11 +0530 Subject: [PATCH 01/17] add emqx boilerplate --- compose/docker-compose.yml | 32 ++++++-- config/config.go | 2 + mq/emqx.go | 156 +++++++++++++++++++++++++++++++++++++ mq/handlers.go | 16 ++++ mq/mq.go | 10 +++ servercfg/serverconf.go | 16 ++++ 6 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 mq/emqx.go diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 16b8dfe3..107bf5e5 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -10,6 +10,8 @@ services: - sqldata:/root/data environment: BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_TYPE: "emqx" + EMQX_REST_ENDPOINT: "http://mq:18083" SERVER_NAME: "NETMAKER_BASE_DOMAIN" STUN_DOMAIN: "stun.NETMAKER_BASE_DOMAIN" SERVER_HOST: "SERVER_PUBLIC_IP" @@ -62,20 +64,34 @@ services: restart: always volumes: - dnsconfig:/root/dnsconfig + # mq: + # container_name: mq + # image: eclipse-mosquitto:2.0.15-openssl + # depends_on: + # - netmaker + # restart: unless-stopped + # command: ["/mosquitto/config/wait.sh"] + # environment: + # MQ_PASSWORD: "REPLACE_MQ_PASSWORD" + # MQ_USERNAME: "REPLACE_MQ_USERNAME" + # volumes: + # - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf + # - /root/wait.sh:/mosquitto/config/wait.sh + # - mosquitto_logs:/mosquitto/log mq: container_name: mq - image: eclipse-mosquitto:2.0.15-openssl + image: emqx/emqx:5.0.17 depends_on: - netmaker restart: unless-stopped - command: ["/mosquitto/config/wait.sh"] environment: - MQ_PASSWORD: "REPLACE_MQ_PASSWORD" - MQ_USERNAME: "REPLACE_MQ_USERNAME" - volumes: - - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf - - /root/wait.sh:/mosquitto/config/wait.sh - - mosquitto_logs:/mosquitto/log + EMQX_NAME: "emqx" + EMQX_DASHBOARD__DEFAULT_PASSWORD: "REPLACE_MQ_PASSWORD" + EMQX_DASHBOARD__DEFAULT_USERNAME: "REPLACE_MQ_USERNAME" + ports: + - "1883:1883" + - "8883:8883" + - "18083:18083" volumes: caddy_data: {} caddy_conf: {} diff --git a/config/config.go b/config/config.go index ec80aba9..11493fb2 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,8 @@ type ServerConfig struct { APIHost string `yaml:"apihost"` APIPort string `yaml:"apiport"` MQHOST string `yaml:"mqhost"` + BrokerType string `yaml:"brokertype` + EmqxRestEndpoint string `yaml:"emqxrestendpoint"` MasterKey string `yaml:"masterkey"` DNSKey string `yaml:"dnskey"` AllowedOrigin string `yaml:"allowedorigin"` diff --git a/mq/emqx.go b/mq/emqx.go new file mode 100644 index 00000000..48b2c159 --- /dev/null +++ b/mq/emqx.go @@ -0,0 +1,156 @@ +package mq + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/gravitl/netmaker/servercfg" +) + +const emqxBrokerType = "emqx" + +type ( + emqxUser struct { + UserID string `json:"user_id"` + Password string `json:"password"` + Admin bool `json:"is_superuser"` + } + + emqxLogin struct { + Username string `json:"username"` + Password string `json:"password"` + } + + emqxLoginResponse struct { + License struct { + Edition string `json:"edition"` + } `json:"license"` + Token string `json:"token"` + Version string `json:"version"` + } +) + +func getEmqxAuthToken() (string, error) { + payload, err := json.Marshal(&emqxLogin{ + Username: servercfg.GetMqUserName(), + Password: servercfg.GetMqPassword(), + }) + if err != nil { + return "", err + } + resp, err := http.Post(servercfg.GetEmqxRestEndpoint()+"/api/v5/login", "application/json", bytes.NewReader(payload)) + if err != nil { + return "", err + } + msg, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("error during EMQX login %v", string(msg)) + } + var loginResp emqxLoginResponse + if err := json.Unmarshal(msg, &loginResp); err != nil { + return "", err + } + return loginResp.Token, nil +} + +// CreateEmqxUser - creates an EMQX user +func CreateEmqxUser(username, password string, admin bool) error { + token, err := getEmqxAuthToken() + if err != nil { + return err + } + payload, err := json.Marshal(&emqxUser{ + UserID: username, + Password: password, + Admin: admin, + }) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, servercfg.GetEmqxRestEndpoint()+"/api/v5/authentication/password_based:built_in_database/users", bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Add("content-type", "application/json") + req.Header.Add("authorization", "Bearer "+token) + resp, err := (&http.Client{}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + msg, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("error creating EMQX user %v", string(msg)) + } + return nil +} + +// DeleteEmqxUser - deletes an EMQX user +func DeleteEmqxUser(username string) error { + token, err := getEmqxAuthToken() + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodDelete, servercfg.GetEmqxRestEndpoint()+"/api/v5/authentication/password_based:built_in_database/users/"+username, nil) + if err != nil { + return err + } + req.Header.Add("authorization", "Bearer "+token) + resp, err := (&http.Client{}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + msg, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("error deleting EMQX user %v", string(msg)) + } + return nil +} + +// CreateEmqxDefaultAuthenticator - creates a default authenticator based on password and using EMQX's built in database as storage +func CreateEmqxDefaultAuthenticator() error { + token, err := getEmqxAuthToken() + if err != nil { + return err + } + payload, err := json.Marshal(&struct { + Mechanism string `json:"mechanism"` + Backend string `json:"backend"` + UserIDType string `json:"user_id_type"` + }{Mechanism: "password_based", Backend: "built_in_database", UserIDType: "username"}) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, servercfg.GetEmqxRestEndpoint()+"/api/v5/authentication", bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Add("content-type", "application/json") + req.Header.Add("authorization", "Bearer "+token) + resp, err := (&http.Client{}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + msg, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("error creating default EMQX authenticator %v", string(msg)) + } + return nil +} diff --git a/mq/handlers.go b/mq/handlers.go index 49555238..16af24da 100644 --- a/mq/handlers.go +++ b/mq/handlers.go @@ -145,6 +145,15 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) { var sendPeerUpdate bool switch hostUpdate.Action { case models.UpdateHost: + if servercfg.GetBrokerType() == emqxBrokerType { + // create EMQX credentials for host if it doesn't exists + if _, err := logic.GetHost(currentHost.ID.String()); err != nil { + if err := CreateEmqxUser(currentHost.ID.String(), currentHost.HostPass, false); err != nil { + logger.Log(0, "failed to add host credentials to EMQX: ", currentHost.ID.String(), err.Error()) + return + } + } + } sendPeerUpdate = logic.UpdateHostFromClient(&hostUpdate.Host, currentHost) err := logic.UpsertHost(currentHost) if err != nil { @@ -152,6 +161,13 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) { return } case models.DeleteHost: + if servercfg.GetBrokerType() == emqxBrokerType { + // delete EMQX credentials for host + if err := DeleteEmqxUser(currentHost.ID.String()); err != nil { + logger.Log(0, "failed to remove host credentials from EMQX: ", currentHost.ID.String(), err.Error()) + return + } + } if err := logic.DisassociateAllNodesFromHost(currentHost.ID.String()); err != nil { logger.Log(0, "failed to delete all nodes of host: ", currentHost.ID.String(), err.Error()) return diff --git a/mq/mq.go b/mq/mq.go index 056dc925..85164acc 100644 --- a/mq/mq.go +++ b/mq/mq.go @@ -2,6 +2,7 @@ package mq import ( "context" + "log" "time" mqtt "github.com/eclipse/paho.mqtt.golang" @@ -38,6 +39,15 @@ func setMqOptions(user, password string, opts *mqtt.ClientOptions) { // SetupMQTT creates a connection to broker and return client func SetupMQTT() { + if servercfg.GetBrokerType() == emqxBrokerType { + // setup authenticator and create admin user + if err := CreateEmqxDefaultAuthenticator(); err != nil { + logger.Log(0, err.Error()) + } + if err := CreateEmqxUser(servercfg.GetMqUserName(), servercfg.GetMqPassword(), true); err != nil { + log.Fatal(err) + } + } opts := mqtt.NewClientOptions() setMqOptions(servercfg.GetMqUserName(), servercfg.GetMqPassword(), opts) opts.SetOnConnectHandler(func(client mqtt.Client) { diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 588259ad..bf39f7ab 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -43,6 +43,8 @@ func GetServerConfig() config.ServerConfig { cfg.NodeID = GetNodeID() cfg.StunHost = GetStunAddr() cfg.StunPort = GetStunPort() + cfg.BrokerType = GetBrokerType() + cfg.EmqxRestEndpoint = GetEmqxRestEndpoint() if IsRestBackend() { cfg.RestBackend = "on" } @@ -250,6 +252,15 @@ func GetMessageQueueEndpoint() (string, bool) { return host + ":" + GetMQServerPort(), secure } +// GetBrokerType - returns the type of MQ broker +func GetBrokerType() string { + if os.Getenv("BROKER_TYPE") != "" { + return os.Getenv("BROKER_TYPE") + } else { + return "mosquitto" + } +} + // GetMasterKey - gets the configured master key of server func GetMasterKey() string { key := "" @@ -613,6 +624,11 @@ func GetMqUserName() string { return password } +// GetEmqxRestEndpoint - returns the REST API Endpoint of EMQX +func GetEmqxRestEndpoint() string { + return os.Getenv("EMQX_REST_ENDPOINT") +} + // IsBasicAuthEnabled - checks if basic auth has been configured to be turned off func IsBasicAuthEnabled() bool { var enabled = true //default From a33ac33462ef41d5f8b1f08da8cc5d9ab15d5428 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 20 Feb 2023 16:17:30 +0530 Subject: [PATCH 02/17] wait for EMQX rest endpoint startup --- compose/docker-compose.yml | 3 +-- mq/mq.go | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 107bf5e5..c7e77ef9 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -39,6 +39,7 @@ services: image: gravitl/netmaker-ui:REPLACE_UI_IMAGE_TAG depends_on: - netmaker + - mq links: - "netmaker:api" restart: always @@ -81,8 +82,6 @@ services: mq: container_name: mq image: emqx/emqx:5.0.17 - depends_on: - - netmaker restart: unless-stopped environment: EMQX_NAME: "emqx" diff --git a/mq/mq.go b/mq/mq.go index 85164acc..b72b4d9d 100644 --- a/mq/mq.go +++ b/mq/mq.go @@ -40,6 +40,7 @@ func setMqOptions(user, password string, opts *mqtt.ClientOptions) { // SetupMQTT creates a connection to broker and return client func SetupMQTT() { if servercfg.GetBrokerType() == emqxBrokerType { + time.Sleep(7 * time.Second) // wait for the REST endpoint to be ready // setup authenticator and create admin user if err := CreateEmqxDefaultAuthenticator(); err != nil { logger.Log(0, err.Error()) From c7a3306d3e36eda2fe54a173169a6acbd8ee788c Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 20 Feb 2023 17:06:27 +0530 Subject: [PATCH 03/17] refresh emqx admin user on server startup --- compose/docker-compose.yml | 9 +++++---- mq/mq.go | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index c7e77ef9..01a6d6f1 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -27,7 +27,7 @@ services: NODE_ID: "netmaker-server-1" MQ_HOST: "mq" MQ_PORT: "443" - MQ_SERVER_PORT: "1883" + MQ_SERVER_PORT: "8083" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" @@ -88,9 +88,10 @@ services: EMQX_DASHBOARD__DEFAULT_PASSWORD: "REPLACE_MQ_PASSWORD" EMQX_DASHBOARD__DEFAULT_USERNAME: "REPLACE_MQ_USERNAME" ports: - - "1883:1883" - - "8883:8883" - - "18083:18083" + - "1883:1883" # MQTT + - "8883:8883" # SSL MQTT + - "8083:8083" # Websockets + - "18083:18083" # Dashboard/REST_API volumes: caddy_data: {} caddy_conf: {} diff --git a/mq/mq.go b/mq/mq.go index b72b4d9d..4ba2f91e 100644 --- a/mq/mq.go +++ b/mq/mq.go @@ -40,11 +40,12 @@ func setMqOptions(user, password string, opts *mqtt.ClientOptions) { // SetupMQTT creates a connection to broker and return client func SetupMQTT() { if servercfg.GetBrokerType() == emqxBrokerType { - time.Sleep(7 * time.Second) // wait for the REST endpoint to be ready + time.Sleep(10 * time.Second) // wait for the REST endpoint to be ready // setup authenticator and create admin user if err := CreateEmqxDefaultAuthenticator(); err != nil { logger.Log(0, err.Error()) } + DeleteEmqxUser(servercfg.GetMqUserName()) if err := CreateEmqxUser(servercfg.GetMqUserName(), servercfg.GetMqPassword(), true); err != nil { log.Fatal(err) } From 0005554dde051db926a421fdd4d09b227972af66 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 20 Feb 2023 17:33:47 +0530 Subject: [PATCH 04/17] update caddyfile --- docker/Caddyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Caddyfile b/docker/Caddyfile index 51a6f98a..c2e6ad0b 100644 --- a/docker/Caddyfile +++ b/docker/Caddyfile @@ -38,5 +38,5 @@ https://stun.NETMAKER_BASE_DOMAIN { # MQ wss://broker.NETMAKER_BASE_DOMAIN { - reverse_proxy ws://mq:8883 + reverse_proxy ws://mq:8083 } From 5279900b7cc901e9ffb0db13bb93d1159db40655 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Tue, 21 Feb 2023 09:12:08 +0530 Subject: [PATCH 05/17] add /mqtt for EMQX websocket listen endpoint --- mq/emqx.go | 2 -- mq/handlers.go | 4 ++-- mq/mq.go | 2 +- servercfg/serverconf.go | 10 +++++++++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mq/emqx.go b/mq/emqx.go index 48b2c159..e2e0b773 100644 --- a/mq/emqx.go +++ b/mq/emqx.go @@ -10,8 +10,6 @@ import ( "github.com/gravitl/netmaker/servercfg" ) -const emqxBrokerType = "emqx" - type ( emqxUser struct { UserID string `json:"user_id"` diff --git a/mq/handlers.go b/mq/handlers.go index 16af24da..af5d93f2 100644 --- a/mq/handlers.go +++ b/mq/handlers.go @@ -145,7 +145,7 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) { var sendPeerUpdate bool switch hostUpdate.Action { case models.UpdateHost: - if servercfg.GetBrokerType() == emqxBrokerType { + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { // create EMQX credentials for host if it doesn't exists if _, err := logic.GetHost(currentHost.ID.String()); err != nil { if err := CreateEmqxUser(currentHost.ID.String(), currentHost.HostPass, false); err != nil { @@ -161,7 +161,7 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) { return } case models.DeleteHost: - if servercfg.GetBrokerType() == emqxBrokerType { + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { // delete EMQX credentials for host if err := DeleteEmqxUser(currentHost.ID.String()); err != nil { logger.Log(0, "failed to remove host credentials from EMQX: ", currentHost.ID.String(), err.Error()) diff --git a/mq/mq.go b/mq/mq.go index 4ba2f91e..86e7d9d6 100644 --- a/mq/mq.go +++ b/mq/mq.go @@ -39,7 +39,7 @@ func setMqOptions(user, password string, opts *mqtt.ClientOptions) { // SetupMQTT creates a connection to broker and return client func SetupMQTT() { - if servercfg.GetBrokerType() == emqxBrokerType { + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { time.Sleep(10 * time.Second) // wait for the REST endpoint to be ready // setup authenticator and create admin user if err := CreateEmqxDefaultAuthenticator(); err != nil { diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index bf39f7ab..6d19e419 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -13,6 +13,9 @@ import ( "github.com/gravitl/netmaker/models" ) +// EmqxBrokerType denotes the broker type for EMQX MQTT +const EmqxBrokerType = "emqx" + var ( Version = "dev" Is_EE = false @@ -249,7 +252,12 @@ func GetMessageQueueEndpoint() (string, bool) { } else { host = "ws://" + host } - return host + ":" + GetMQServerPort(), secure + host += ":" + GetMQServerPort() + // websocket listen endpoint for EMQX broker is ws://host:port/mqtt + if GetBrokerType() == EmqxBrokerType { + host += "/mqtt" + } + return host, secure } // GetBrokerType - returns the type of MQ broker From 186064052c38e120bfa61d8e0ce48a91cbee8281 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Tue, 21 Feb 2023 09:24:41 +0530 Subject: [PATCH 06/17] fix broker type yaml tag --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 11493fb2..f84a4ccc 100644 --- a/config/config.go +++ b/config/config.go @@ -37,7 +37,7 @@ type ServerConfig struct { APIHost string `yaml:"apihost"` APIPort string `yaml:"apiport"` MQHOST string `yaml:"mqhost"` - BrokerType string `yaml:"brokertype` + BrokerType string `yaml:"brokertype"` EmqxRestEndpoint string `yaml:"emqxrestendpoint"` MasterKey string `yaml:"masterkey"` DNSKey string `yaml:"dnskey"` From 5e056dd7ec1050ebdf294ec5df6f07828a2b5a60 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Wed, 22 Feb 2023 12:35:39 +0530 Subject: [PATCH 07/17] update docker compose --- compose/docker-compose-emqx.yml | 85 +++++++++++++++++++++++++++++++++ compose/docker-compose.ee.yml | 25 +++++----- compose/docker-compose.yml | 38 +++++---------- docker/Caddyfile | 2 +- 4 files changed, 109 insertions(+), 41 deletions(-) create mode 100644 compose/docker-compose-emqx.yml diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml new file mode 100644 index 00000000..c6e57beb --- /dev/null +++ b/compose/docker-compose-emqx.yml @@ -0,0 +1,85 @@ +version: "3.4" + +services: + netmaker: + container_name: netmaker + image: gravitl/netmaker:v0.18.1 + restart: always + volumes: + - dnsconfig:/root/config/dnsconfig + - sqldata:/root/data + environment: + BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_TYPE: "emqx" + EMQX_REST_ENDPOINT: "http://mq:18083" + SERVER_NAME: "NETMAKER_BASE_DOMAIN" + STUN_DOMAIN: "stun.NETMAKER_BASE_DOMAIN" + SERVER_HOST: "SERVER_PUBLIC_IP" + SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443" + COREDNS_ADDR: "SERVER_PUBLIC_IP" + DNS_MODE: "on" + SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN" + API_PORT: "8081" + MASTER_KEY: "REPLACE_MASTER_KEY" + CORS_ALLOWED_ORIGIN: "*" + DISPLAY_KEYS: "on" + DATABASE: "sqlite" + NODE_ID: "netmaker-server-1" + MQ_HOST: "mq" + MQ_PORT: "443" + MQ_SERVER_PORT: "8083" + STUN_PORT: "3478" + VERBOSITY: "1" + MQ_PASSWORD: "REPLACE_MQ_PASSWORD" + MQ_USERNAME: "REPLACE_MQ_USERNAME" + ports: + - "3478:3478/udp" + netmaker-ui: + container_name: netmaker-ui + image: gravitl/netmaker-ui:v0.18.1 + depends_on: + - netmaker + links: + - "netmaker:api" + restart: always + environment: + BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN" + caddy: + image: caddy:2.6.2 + container_name: caddy + restart: unless-stopped + volumes: + - /root/Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_conf:/config + ports: + - "80:80" + - "443:443" + coredns: + container_name: coredns + image: coredns/coredns + command: -conf /root/dnsconfig/Corefile + depends_on: + - netmaker + restart: always + volumes: + - dnsconfig:/root/dnsconfig + mq: + container_name: mq + image: emqx/emqx:5.0.17 + restart: unless-stopped + environment: + EMQX_NAME: "emqx" + EMQX_DASHBOARD__DEFAULT_PASSWORD: "REPLACE_MQ_PASSWORD" + EMQX_DASHBOARD__DEFAULT_USERNAME: "REPLACE_MQ_USERNAME" + ports: + - "1883:1883" # MQTT + - "8883:8883" # SSL MQTT + - "8083:8083" # Websockets + - "18083:18083" # Dashboard/REST_API +volumes: + caddy_data: {} + caddy_conf: {} + sqldata: {} + dnsconfig: {} + mosquitto_logs: {} diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index fee3a412..e846244e 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -10,6 +10,8 @@ services: - sqldata:/root/data environment: BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_TYPE: "emqx" + EMQX_REST_ENDPOINT: "http://mq:18083" SERVER_NAME: "NETMAKER_BASE_DOMAIN" STUN_DOMAIN: "stun.NETMAKER_BASE_DOMAIN" SERVER_HOST: "SERVER_PUBLIC_IP" @@ -25,7 +27,8 @@ services: NODE_ID: "netmaker-server-1" MQ_HOST: "mq" MQ_PORT: "443" - MQ_SERVER_PORT: "1883" + MQ_SERVER_PORT: "8083" + MQ_PASSWORD: "REPLACE_MQ_PASSWORD" MQ_USERNAME: "REPLACE_MQ_USERNAME" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" STUN_PORT: "3478" @@ -67,21 +70,17 @@ services: - dnsconfig:/root/dnsconfig mq: container_name: mq - image: eclipse-mosquitto:2.0.15-openssl - depends_on: - - netmaker + image: emqx/emqx:5.0.17 restart: unless-stopped - command: ["/mosquitto/config/wait.sh"] environment: - MQ_PASSWORD: "REPLACE_MQ_PASSWORD" - MQ_USERNAME: "REPLACE_MQ_USERNAME" - volumes: - - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf - - /root/wait.sh:/mosquitto/config/wait.sh - - mosquitto_logs:/mosquitto/log + EMQX_NAME: "emqx" + EMQX_DASHBOARD__DEFAULT_PASSWORD: "REPLACE_MQ_PASSWORD" + EMQX_DASHBOARD__DEFAULT_USERNAME: "REPLACE_MQ_USERNAME" ports: - - "1883:1883" - - "8883:8883" + - "1883:1883" # MQTT + - "8883:8883" # SSL MQTT + - "8083:8083" # Websockets + - "18083:18083" # Dashboard/REST_API prometheus: container_name: prometheus image: gravitl/netmaker-prometheus:latest diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 01a6d6f1..16b8dfe3 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -10,8 +10,6 @@ services: - sqldata:/root/data environment: BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" - BROKER_TYPE: "emqx" - EMQX_REST_ENDPOINT: "http://mq:18083" SERVER_NAME: "NETMAKER_BASE_DOMAIN" STUN_DOMAIN: "stun.NETMAKER_BASE_DOMAIN" SERVER_HOST: "SERVER_PUBLIC_IP" @@ -27,7 +25,7 @@ services: NODE_ID: "netmaker-server-1" MQ_HOST: "mq" MQ_PORT: "443" - MQ_SERVER_PORT: "8083" + MQ_SERVER_PORT: "1883" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" @@ -39,7 +37,6 @@ services: image: gravitl/netmaker-ui:REPLACE_UI_IMAGE_TAG depends_on: - netmaker - - mq links: - "netmaker:api" restart: always @@ -65,33 +62,20 @@ services: restart: always volumes: - dnsconfig:/root/dnsconfig - # mq: - # container_name: mq - # image: eclipse-mosquitto:2.0.15-openssl - # depends_on: - # - netmaker - # restart: unless-stopped - # command: ["/mosquitto/config/wait.sh"] - # environment: - # MQ_PASSWORD: "REPLACE_MQ_PASSWORD" - # MQ_USERNAME: "REPLACE_MQ_USERNAME" - # volumes: - # - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf - # - /root/wait.sh:/mosquitto/config/wait.sh - # - mosquitto_logs:/mosquitto/log mq: container_name: mq - image: emqx/emqx:5.0.17 + image: eclipse-mosquitto:2.0.15-openssl + depends_on: + - netmaker restart: unless-stopped + command: ["/mosquitto/config/wait.sh"] environment: - EMQX_NAME: "emqx" - EMQX_DASHBOARD__DEFAULT_PASSWORD: "REPLACE_MQ_PASSWORD" - EMQX_DASHBOARD__DEFAULT_USERNAME: "REPLACE_MQ_USERNAME" - ports: - - "1883:1883" # MQTT - - "8883:8883" # SSL MQTT - - "8083:8083" # Websockets - - "18083:18083" # Dashboard/REST_API + MQ_PASSWORD: "REPLACE_MQ_PASSWORD" + MQ_USERNAME: "REPLACE_MQ_USERNAME" + volumes: + - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf + - /root/wait.sh:/mosquitto/config/wait.sh + - mosquitto_logs:/mosquitto/log volumes: caddy_data: {} caddy_conf: {} diff --git a/docker/Caddyfile b/docker/Caddyfile index c2e6ad0b..d8392b7d 100644 --- a/docker/Caddyfile +++ b/docker/Caddyfile @@ -38,5 +38,5 @@ https://stun.NETMAKER_BASE_DOMAIN { # MQ wss://broker.NETMAKER_BASE_DOMAIN { - reverse_proxy ws://mq:8083 + reverse_proxy ws://mq:8883 # For EMQX websockets use `reverse_proxy ws://mq:8083` } From 92327ebd2f94c89abc92ab90d06cb67b4a9f4afa Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Wed, 22 Feb 2023 20:45:13 +0530 Subject: [PATCH 08/17] update mq host specification --- compose/docker-compose-emqx.yml | 3 +-- compose/docker-compose.ee.yml | 3 +-- compose/docker-compose.reference.yml | 3 +-- compose/docker-compose.yml | 3 +-- servercfg/serverconf.go | 24 ++---------------------- 5 files changed, 6 insertions(+), 30 deletions(-) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index c6e57beb..aa079478 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -25,9 +25,8 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "mq" + MQ_HOST: "ws://mq:8083/mqtt" MQ_PORT: "443" - MQ_SERVER_PORT: "8083" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index e846244e..304a5bfb 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -25,9 +25,8 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "mq" + MQ_HOST: "ws://mq:8083/mqtt" MQ_PORT: "443" - MQ_SERVER_PORT: "8083" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" MQ_USERNAME: "REPLACE_MQ_USERNAME" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" diff --git a/compose/docker-compose.reference.yml b/compose/docker-compose.reference.yml index ed51f789..c8f6931a 100644 --- a/compose/docker-compose.reference.yml +++ b/compose/docker-compose.reference.yml @@ -26,9 +26,8 @@ services: DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display. DATABASE: "sqlite" # Database to use - sqlite, postgres, or rqlite NODE_ID: "netmaker-server-1" # used for HA - identifies this server vs other servers - MQ_HOST: "mq" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. + MQ_HOST: ""ws://mq:1883"" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. MQ_PORT: "443" # the reachable port of MQ - change if external MQ port changes (port on proxy, not necessarily the one exposed in docker-compose) - MQ_SERVER_PORT: "1883" # the reachable port of MQ by the server - change if internal MQ port changes (or use external port if MQ is not on the same machine) MQ_USERNAME: "REPLACE_MQ_USERNAME" # the username to set for MQ access MQ_PASSWORD: "REPLACE_MQ_PASSWORD" # the password to set for MQ access STUN_PORT: "3478" # the reachable port of STUN on the server diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 16b8dfe3..6a3d4772 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -23,9 +23,8 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "mq" + MQ_HOST: "ws://mq:1883" MQ_PORT: "443" - MQ_SERVER_PORT: "1883" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 6d19e419..14a33ae0 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -245,19 +245,10 @@ func GetMessageQueueEndpoint() (string, bool) { host = os.Getenv("MQ_HOST") } else if config.Config.Server.MQHOST != "" { host = config.Config.Server.MQHOST - } - secure := strings.Contains(host, "wss") || strings.Contains(host, "ssl") - if secure { - host = "wss://" + host } else { - host = "ws://" + host + host += ":1883" // default } - host += ":" + GetMQServerPort() - // websocket listen endpoint for EMQX broker is ws://host:port/mqtt - if GetBrokerType() == EmqxBrokerType { - host += "/mqtt" - } - return host, secure + return host, strings.Contains(host, "wss") || strings.Contains(host, "ssl") } // GetBrokerType - returns the type of MQ broker @@ -599,17 +590,6 @@ func GetAzureTenant() string { return azureTenant } -// GetMQServerPort - get mq port for server -func GetMQServerPort() string { - port := "1883" //default - if os.Getenv("MQ_SERVER_PORT") != "" { - port = os.Getenv("MQ_SERVER_PORT") - } else if config.Config.Server.MQServerPort != "" { - port = config.Config.Server.MQServerPort - } - return port -} - // GetMqPassword - fetches the MQ password func GetMqPassword() string { password := "" From 55042669920d14d0a2d2c638741d926670686832 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Wed, 22 Feb 2023 20:48:20 +0530 Subject: [PATCH 09/17] remove duplicate env --- compose/docker-compose.ee.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index 304a5bfb..7cd29709 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -27,7 +27,6 @@ services: NODE_ID: "netmaker-server-1" MQ_HOST: "ws://mq:8083/mqtt" MQ_PORT: "443" - MQ_PASSWORD: "REPLACE_MQ_PASSWORD" MQ_USERNAME: "REPLACE_MQ_USERNAME" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" STUN_PORT: "3478" From 6f8f9ec5eff1ecaf0ae96c57348a7e5627b95ee9 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Wed, 22 Feb 2023 22:02:43 +0530 Subject: [PATCH 10/17] rename MQ_HOST -> BROKER_ENDPOINT --- compose/docker-compose-emqx.yml | 2 +- compose/docker-compose.ee.yml | 4 ++-- compose/docker-compose.reference.yml | 2 +- compose/docker-compose.yml | 2 +- servercfg/serverconf.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index aa079478..e2287bb5 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -25,7 +25,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "ws://mq:8083/mqtt" + BROKER_ENDPOINT: "ws://mq:8083/mqtt" MQ_PORT: "443" STUN_PORT: "3478" VERBOSITY: "1" diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index 7cd29709..6ec9a78b 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -25,7 +25,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "ws://mq:8083/mqtt" + BROKER_ENDPOINT: "ws://mq:8083/mqtt" MQ_PORT: "443" MQ_USERNAME: "REPLACE_MQ_USERNAME" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" @@ -112,7 +112,7 @@ services: depends_on: - netmaker environment: - MQ_HOST: "mq" + BROKER_ENDPOINT: "mq" MQ_PORT: "443" MQ_SERVER_PORT: "1883" PROMETHEUS: "on" diff --git a/compose/docker-compose.reference.yml b/compose/docker-compose.reference.yml index c8f6931a..9715818f 100644 --- a/compose/docker-compose.reference.yml +++ b/compose/docker-compose.reference.yml @@ -26,7 +26,7 @@ services: DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display. DATABASE: "sqlite" # Database to use - sqlite, postgres, or rqlite NODE_ID: "netmaker-server-1" # used for HA - identifies this server vs other servers - MQ_HOST: ""ws://mq:1883"" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. + BROKER_ENDPOINT: ""ws://mq:1883"" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. MQ_PORT: "443" # the reachable port of MQ - change if external MQ port changes (port on proxy, not necessarily the one exposed in docker-compose) MQ_USERNAME: "REPLACE_MQ_USERNAME" # the username to set for MQ access MQ_PASSWORD: "REPLACE_MQ_PASSWORD" # the password to set for MQ access diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 6a3d4772..a6cc2d13 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -23,7 +23,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - MQ_HOST: "ws://mq:1883" + BROKER_ENDPOINT: "ws://mq:1883" MQ_PORT: "443" STUN_PORT: "3478" VERBOSITY: "1" diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 14a33ae0..9a1fede1 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -241,8 +241,8 @@ func GetMQPort() string { // GetMessageQueueEndpoint - gets the message queue endpoint func GetMessageQueueEndpoint() (string, bool) { host, _ := GetPublicIP() - if os.Getenv("MQ_HOST") != "" { - host = os.Getenv("MQ_HOST") + if os.Getenv("BROKER_ENDPOINT") != "" { + host = os.Getenv("BROKER_ENDPOINT") } else if config.Config.Server.MQHOST != "" { host = config.Config.Server.MQHOST } else { From 2700acc1d755f7283a18123a696f1692d833f7b4 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Thu, 23 Feb 2023 17:48:37 +0530 Subject: [PATCH 11/17] user BROKER_ENDPOINT and SERVER_BROKER_ENDPOINT --- compose/docker-compose-emqx.yml | 5 ++-- compose/docker-compose.ee.yml | 8 +++---- compose/docker-compose.reference.yml | 5 ++-- compose/docker-compose.yml | 5 ++-- config/config.go | 5 ++-- k8s/server/netmaker-server.yaml | 8 ++----- servercfg/serverconf.go | 35 ++++++---------------------- 7 files changed, 20 insertions(+), 51 deletions(-) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index e2287bb5..67b56fb5 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -9,7 +9,7 @@ services: - dnsconfig:/root/config/dnsconfig - sqldata:/root/data environment: - BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN/mqtt" BROKER_TYPE: "emqx" EMQX_REST_ENDPOINT: "http://mq:18083" SERVER_NAME: "NETMAKER_BASE_DOMAIN" @@ -25,8 +25,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - BROKER_ENDPOINT: "ws://mq:8083/mqtt" - MQ_PORT: "443" + SERVER_BROKER_ENDPOINT: "ws://mq:8083/mqtt" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index 6ec9a78b..689065f7 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -9,7 +9,7 @@ services: - dnsconfig:/root/config/dnsconfig - sqldata:/root/data environment: - BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN/mqtt" BROKER_TYPE: "emqx" EMQX_REST_ENDPOINT: "http://mq:18083" SERVER_NAME: "NETMAKER_BASE_DOMAIN" @@ -25,8 +25,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - BROKER_ENDPOINT: "ws://mq:8083/mqtt" - MQ_PORT: "443" + SERVER_BROKER_ENDPOINT: "ws://mq:8083/mqtt" MQ_USERNAME: "REPLACE_MQ_USERNAME" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" STUN_PORT: "3478" @@ -112,8 +111,7 @@ services: depends_on: - netmaker environment: - BROKER_ENDPOINT: "mq" - MQ_PORT: "443" + SERVER_BROKER_ENDPOINT: "mq" MQ_SERVER_PORT: "1883" PROMETHEUS: "on" VERBOSITY: "1" diff --git a/compose/docker-compose.reference.yml b/compose/docker-compose.reference.yml index 9715818f..a67c2335 100644 --- a/compose/docker-compose.reference.yml +++ b/compose/docker-compose.reference.yml @@ -10,7 +10,7 @@ services: - sqldata:/root/data - shared_certs:/etc/netmaker environment: # Necessary capabilities to set iptables when running in container - BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" # The domain/host IP indicating the mq broker address + BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN" # The domain/host IP indicating the mq broker address SERVER_NAME: "NETMAKER_BASE_DOMAIN" # The base domain of netmaker SERVER_HOST: "SERVER_PUBLIC_IP" # Set to public IP of machine. SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN" # Overrides SERVER_HOST if set. Useful for making HTTP available via different interfaces/networks. @@ -26,8 +26,7 @@ services: DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display. DATABASE: "sqlite" # Database to use - sqlite, postgres, or rqlite NODE_ID: "netmaker-server-1" # used for HA - identifies this server vs other servers - BROKER_ENDPOINT: ""ws://mq:1883"" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. - MQ_PORT: "443" # the reachable port of MQ - change if external MQ port changes (port on proxy, not necessarily the one exposed in docker-compose) + SERVER_BROKER_ENDPOINT: ""ws://mq:1883"" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container. MQ_USERNAME: "REPLACE_MQ_USERNAME" # the username to set for MQ access MQ_PASSWORD: "REPLACE_MQ_PASSWORD" # the password to set for MQ access STUN_PORT: "3478" # the reachable port of STUN on the server diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index a6cc2d13..a9937df6 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -9,7 +9,7 @@ services: - dnsconfig:/root/config/dnsconfig - sqldata:/root/data environment: - BROKER_NAME: "broker.NETMAKER_BASE_DOMAIN" + BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN" SERVER_NAME: "NETMAKER_BASE_DOMAIN" STUN_DOMAIN: "stun.NETMAKER_BASE_DOMAIN" SERVER_HOST: "SERVER_PUBLIC_IP" @@ -23,8 +23,7 @@ services: DISPLAY_KEYS: "on" DATABASE: "sqlite" NODE_ID: "netmaker-server-1" - BROKER_ENDPOINT: "ws://mq:1883" - MQ_PORT: "443" + SERVER_BROKER_ENDPOINT: "ws://mq:1883" STUN_PORT: "3478" VERBOSITY: "1" MQ_PASSWORD: "REPLACE_MQ_PASSWORD" diff --git a/config/config.go b/config/config.go index f84a4ccc..944c8bb2 100644 --- a/config/config.go +++ b/config/config.go @@ -36,7 +36,8 @@ type ServerConfig struct { APIConnString string `yaml:"apiconn"` APIHost string `yaml:"apihost"` APIPort string `yaml:"apiport"` - MQHOST string `yaml:"mqhost"` + BrokerEndpoint string `yaml:"brokerendpoint"` + ServerBrokerEndpoint string `yaml:"serverbrokerendpoint"` BrokerType string `yaml:"brokertype"` EmqxRestEndpoint string `yaml:"emqxrestendpoint"` MasterKey string `yaml:"masterkey"` @@ -64,8 +65,6 @@ type ServerConfig struct { AzureTenant string `yaml:"azuretenant"` Telemetry string `yaml:"telemetry"` HostNetwork string `yaml:"hostnetwork"` - MQPort string `yaml:"mqport"` - MQServerPort string `yaml:"mqserverport"` Server string `yaml:"server"` Broker string `yam:"broker"` PublicIPService string `yaml:"publicipservice"` diff --git a/k8s/server/netmaker-server.yaml b/k8s/server/netmaker-server.yaml index a73ddb4d..516316c7 100644 --- a/k8s/server/netmaker-server.yaml +++ b/k8s/server/netmaker-server.yaml @@ -71,12 +71,8 @@ spec: value: REPLACE_MASTER_KEY - name: CORS_ALLOWED_ORIGIN value: '*' - - name: MQ_HOST - value: "mq" - - name: MQ_PORT - value: "443" - - name: MQ_SERVER_PORT - value: "1883" + - name: SERVER_BROKER_ENDPOINT + value: "ws://mq:1883" - name: PLATFORM value: "Kubernetes" - name: VERBOSITY diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 9a1fede1..774ddef6 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -38,7 +38,6 @@ func GetServerConfig() config.ServerConfig { cfg.CoreDNSAddr = GetCoreDNSAddr() cfg.APIHost = GetAPIHost() cfg.APIPort = GetAPIPort() - cfg.MQPort = GetMQPort() cfg.MasterKey = "(hidden)" cfg.DNSKey = "(hidden)" cfg.AllowedOrigin = GetAllowedOrigin() @@ -92,13 +91,11 @@ func GetServerConfig() config.ServerConfig { func GetServerInfo() models.ServerConfig { var cfg models.ServerConfig cfg.Server = GetServer() - cfg.Broker = GetBroker() cfg.MQUserName = GetMqUserName() cfg.MQPassword = GetMqPassword() cfg.API = GetAPIConnString() cfg.CoreDNSAddr = GetCoreDNSAddr() cfg.APIPort = GetAPIPort() - cfg.MQPort = GetMQPort() cfg.DNSMode = "off" if IsDNSMode() { cfg.DNSMode = "on" @@ -227,24 +224,17 @@ func GetCoreDNSAddr() string { return addr } -// GetMQPort - gets the mq port -func GetMQPort() string { - port := "8883" //default - if os.Getenv("MQ_PORT") != "" { - port = os.Getenv("MQ_PORT") - } else if config.Config.Server.MQPort != "" { - port = config.Config.Server.MQPort - } - return port -} - // GetMessageQueueEndpoint - gets the message queue endpoint func GetMessageQueueEndpoint() (string, bool) { host, _ := GetPublicIP() - if os.Getenv("BROKER_ENDPOINT") != "" { + if os.Getenv("SERVER_BROKER_ENDPOINT") != "" { + host = os.Getenv("SERVER_BROKER_ENDPOINT") + } else if config.Config.Server.ServerBrokerEndpoint != "" { + host = config.Config.Server.ServerBrokerEndpoint + } else if os.Getenv("BROKER_ENDPOINT") != "" { host = os.Getenv("BROKER_ENDPOINT") - } else if config.Config.Server.MQHOST != "" { - host = config.Config.Server.MQHOST + } else if config.Config.Server.BrokerEndpoint != "" { + host = config.Config.Server.BrokerEndpoint } else { host += ":1883" // default } @@ -376,17 +366,6 @@ func GetServer() string { return server } -// GetBroker - gets the broker name -func GetBroker() string { - server := "" - if os.Getenv("BROKER_NAME") != "" { - server = os.Getenv("BROKER_NAME") - } else if config.Config.Server.Broker != "" { - server = config.Config.Server.Broker - } - return server -} - func GetVerbosity() int32 { var verbosity = 0 var err error From 8dbd7d109eddd69e676a0153c50f66f773aa6411 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Thu, 23 Feb 2023 17:52:58 +0530 Subject: [PATCH 12/17] update missing config --- compose/docker-compose.ee.yml | 4 ++-- k8s/server/netmaker-server.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose.ee.yml b/compose/docker-compose.ee.yml index 689065f7..d5567772 100644 --- a/compose/docker-compose.ee.yml +++ b/compose/docker-compose.ee.yml @@ -111,8 +111,8 @@ services: depends_on: - netmaker environment: - SERVER_BROKER_ENDPOINT: "mq" - MQ_SERVER_PORT: "1883" + SERVER_BROKER_ENDPOINT: "ws://mq:8083/mqtt" + BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN/mqtt" PROMETHEUS: "on" VERBOSITY: "1" API_PORT: "8085" diff --git a/k8s/server/netmaker-server.yaml b/k8s/server/netmaker-server.yaml index 516316c7..16caa4d8 100644 --- a/k8s/server/netmaker-server.yaml +++ b/k8s/server/netmaker-server.yaml @@ -73,6 +73,8 @@ spec: value: '*' - name: SERVER_BROKER_ENDPOINT value: "ws://mq:1883" + - name: BROKER_ENDPOINT + value: "wss://broker.NETMAKER_BASE_DOMAIN" - name: PLATFORM value: "Kubernetes" - name: VERBOSITY From 314af6b51debff223b2bacb231f798b4fa0586a1 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 27 Feb 2023 17:18:35 +0530 Subject: [PATCH 13/17] create emqx credentials for host upon creation --- controllers/node.go | 8 +++++++- mq/handlers.go | 9 --------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/controllers/node.go b/controllers/node.go index 6e91b48e..321d2135 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -564,6 +564,13 @@ func createNode(w http.ResponseWriter, r *http.Request) { data.Node.Server = servercfg.GetServer() if !logic.HostExists(&data.Host) { logic.CheckHostPorts(&data.Host) + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { + // create EMQX credentials for host if it doesn't exists + if err := mq.CreateEmqxUser(data.Host.ID.String(), data.Host.HostPass, false); err != nil { + logger.Log(0, "failed to add host credentials to EMQX: ", data.Host.ID.String(), err.Error()) + return + } + } } if err := logic.CreateHost(&data.Host); err != nil { if errors.Is(err, logic.ErrHostExists) { @@ -589,7 +596,6 @@ func createNode(w http.ResponseWriter, r *http.Request) { return } } - err = logic.AssociateNodeToHost(&data.Node, &data.Host) if err != nil { logger.Log(0, r.Header.Get("user"), diff --git a/mq/handlers.go b/mq/handlers.go index af5d93f2..ed15da19 100644 --- a/mq/handlers.go +++ b/mq/handlers.go @@ -145,15 +145,6 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) { var sendPeerUpdate bool switch hostUpdate.Action { case models.UpdateHost: - if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { - // create EMQX credentials for host if it doesn't exists - if _, err := logic.GetHost(currentHost.ID.String()); err != nil { - if err := CreateEmqxUser(currentHost.ID.String(), currentHost.HostPass, false); err != nil { - logger.Log(0, "failed to add host credentials to EMQX: ", currentHost.ID.String(), err.Error()) - return - } - } - } sendPeerUpdate = logic.UpdateHostFromClient(&hostUpdate.Host, currentHost) err := logic.UpsertHost(currentHost) if err != nil { From b803e64ee42dc3fed62cd5a57274fac73218f335 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 27 Feb 2023 17:52:36 +0530 Subject: [PATCH 14/17] return broker endpoint --- servercfg/serverconf.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 774ddef6..fb9503b1 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -97,6 +97,7 @@ func GetServerInfo() models.ServerConfig { cfg.CoreDNSAddr = GetCoreDNSAddr() cfg.APIPort = GetAPIPort() cfg.DNSMode = "off" + cfg.Broker = GetPublicBrokerEndpoint() if IsDNSMode() { cfg.DNSMode = "on" } @@ -224,6 +225,15 @@ func GetCoreDNSAddr() string { return addr } +// GetPublicBrokerEndpoint - returns the public broker endpoint which shall be used by netclient +func GetPublicBrokerEndpoint() string { + if os.Getenv("BROKER_ENDPOINT") != "" { + return os.Getenv("BROKER_ENDPOINT") + } else { + return config.Config.Server.BrokerEndpoint + } +} + // GetMessageQueueEndpoint - gets the message queue endpoint func GetMessageQueueEndpoint() (string, bool) { host, _ := GetPublicIP() From c57c8921198ecb1d1ae815224fbea7eca53cdb3d Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Mon, 27 Feb 2023 21:19:59 +0530 Subject: [PATCH 15/17] remove duplicate env var --- config/config.go | 3 +-- servercfg/serverconf.go | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 944c8bb2..d26db1e9 100644 --- a/config/config.go +++ b/config/config.go @@ -36,7 +36,7 @@ type ServerConfig struct { APIConnString string `yaml:"apiconn"` APIHost string `yaml:"apihost"` APIPort string `yaml:"apiport"` - BrokerEndpoint string `yaml:"brokerendpoint"` + Broker string `yam:"broker"` ServerBrokerEndpoint string `yaml:"serverbrokerendpoint"` BrokerType string `yaml:"brokertype"` EmqxRestEndpoint string `yaml:"emqxrestendpoint"` @@ -66,7 +66,6 @@ type ServerConfig struct { Telemetry string `yaml:"telemetry"` HostNetwork string `yaml:"hostnetwork"` Server string `yaml:"server"` - Broker string `yam:"broker"` PublicIPService string `yaml:"publicipservice"` MQPassword string `yaml:"mqpassword"` MQUserName string `yaml:"mqusername"` diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index fb9503b1..1bcd5081 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -230,7 +230,7 @@ func GetPublicBrokerEndpoint() string { if os.Getenv("BROKER_ENDPOINT") != "" { return os.Getenv("BROKER_ENDPOINT") } else { - return config.Config.Server.BrokerEndpoint + return config.Config.Server.Broker } } @@ -243,8 +243,8 @@ func GetMessageQueueEndpoint() (string, bool) { host = config.Config.Server.ServerBrokerEndpoint } else if os.Getenv("BROKER_ENDPOINT") != "" { host = os.Getenv("BROKER_ENDPOINT") - } else if config.Config.Server.BrokerEndpoint != "" { - host = config.Config.Server.BrokerEndpoint + } else if config.Config.Server.Broker != "" { + host = config.Config.Server.Broker } else { host += ":1883" // default } From bf793a54120edd05c26eb4376eaa209408fad85c Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Tue, 28 Feb 2023 09:29:49 +0530 Subject: [PATCH 16/17] add mqtts for secure check --- servercfg/serverconf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 1bcd5081..e3fe8764 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -248,7 +248,7 @@ func GetMessageQueueEndpoint() (string, bool) { } else { host += ":1883" // default } - return host, strings.Contains(host, "wss") || strings.Contains(host, "ssl") + return host, strings.Contains(host, "wss") || strings.Contains(host, "ssl") || strings.Contains(host, "mqtts") } // GetBrokerType - returns the type of MQ broker From 6d942be9a0520b56333dbaac9a197ed8701fd416 Mon Sep 17 00:00:00 2001 From: Anish Mukherjee Date: Tue, 28 Feb 2023 11:04:31 +0530 Subject: [PATCH 17/17] update versions --- compose/docker-compose-emqx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index 67b56fb5..b599b009 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -3,7 +3,7 @@ version: "3.4" services: netmaker: container_name: netmaker - image: gravitl/netmaker:v0.18.1 + image: gravitl/netmaker:v0.18.2 restart: always volumes: - dnsconfig:/root/config/dnsconfig @@ -34,7 +34,7 @@ services: - "3478:3478/udp" netmaker-ui: container_name: netmaker-ui - image: gravitl/netmaker-ui:v0.18.1 + image: gravitl/netmaker-ui:v0.18.2 depends_on: - netmaker links: