gravitl/netmaker
1
1
Fork 0
mirror of https://github.com/gravitl/netmaker.git synced 2026-02-18 20:05:05 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #197 from gravitl/feature_v0.6_kube

Browse source 
Feature v0.6 kube
This commit is contained in:
Alex 2021-07-12 12:11:51 -07:00 • committed by GitHub
commit e505efb27a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 644 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
config/config.go
View file

@ -36,10 +36,13 @@ type EnvironmentConfig struct {
// ServerConfig :
type ServerConfig struct {
APIConnString string `yaml:"apiconn"`
APIHost string `yaml:"apihost"`
APIPort string `yaml:"apiport"`
GRPCConnString string `yaml:"grpcconn"`
GRPCHost string `yaml:"grpchost"`
GRPCPort string `yaml:"grpcport"`
GRPCSecure string `yaml:"grpcsecure"`
DefaultNodeLimit int32 `yaml:"defaultnodelimit"`
MasterKey string `yaml:"masterkey"`
AllowedOrigin string `yaml:"allowedorigin"`
@ -48,6 +51,8 @@ type ServerConfig struct {
ClientMode string `yaml:"clientmode"`
DNSMode string `yaml:"dnsmode"`
DisableRemoteIPCheck string `yaml:"disableremoteipcheck"`
DisableDefaultNet string `yaml:"disabledefaultnet"`
GRPCSSL string `yaml:"grpcssl"`
}
type WG struct {

2
controllers/authGrpc.go
View file

@ -84,7 +84,7 @@ func grpcAuthorize(ctx context.Context) error {
}
emptynode := models.Node{}
node, err := functions.GetNodeByMacAddress(network, mac)
if err != nil || node == emptynode {
if err != nil || node.MacAddress == emptynode.MacAddress {
return status.Errorf(codes.Unauthenticated, "Node does not exist.")
}

88
controllers/networkHttpController.go
View file

@ -6,11 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"strings"
"time"
"github.com/go-playground/validator/v10"
"github.com/gorilla/mux"
"github.com/gravitl/netmaker/functions"
@ -139,6 +137,14 @@ func RemoveComms(networks []models.Network) []models.Network {
func ValidateNetworkUpdate(network models.NetworkUpdate) error {
v := validator.New()
_ = v.RegisterValidation("netid_valid", func(fl validator.FieldLevel) bool {
if fl.Field().String() == "" {
return true
}
inCharSet := functions.NameInNetworkCharSet(fl.Field().String())
return inCharSet
})
// _ = v.RegisterValidation("addressrange_valid", func(fl validator.FieldLevel) bool {
// isvalid := fl.Field().String() == "" || functions.IsIpCIDR(fl.Field().String())
// return isvalid
@ -194,13 +200,14 @@ func ValidateNetworkCreate(network models.Network) error {
//
_ = v.RegisterValidation("netid_valid", func(fl validator.FieldLevel) bool {
isFieldUnique, _ := functions.IsNetworkNameUnique(fl.Field().String())
// inCharSet := functions.NameInNetworkCharSet(fl.Field().String())
return isFieldUnique
inCharSet := functions.NameInNetworkCharSet(fl.Field().String())
return isFieldUnique && inCharSet
})
//
_ = v.RegisterValidation("displayname_unique", func(fl validator.FieldLevel) bool {
_ = v.RegisterValidation("displayname_valid", func(fl validator.FieldLevel) bool {
isFieldUnique, _ := functions.IsNetworkDisplayNameUnique(fl.Field().String())
return isFieldUnique
inCharSet := functions.NameInNetworkCharSet(fl.Field().String())
return isFieldUnique && inCharSet
})
err := v.Struct(network)
@ -677,12 +684,40 @@ func CreateAccessKey(accesskey models.AccessKey, network models.Network) (models
}
netID := network.NetID
grpcaddress := net.JoinHostPort(servercfg.GetGRPCHost(), servercfg.GetGRPCPort())
apiaddress := net.JoinHostPort(servercfg.GetAPIHost(), servercfg.GetAPIPort())
wgport := servercfg.GetGRPCWGPort()
accessstringdec := wgport + "|" +grpcaddress + "|" + apiaddress + "|" + netID + "|" + accesskey.Value + "|" + privAddr
accesskey.AccessString = base64.StdEncoding.EncodeToString([]byte(accessstringdec))
var accessToken models.AccessToken
s := servercfg.GetServerConfig()
w := servercfg.GetWGConfig()
servervals := models.ServerConfig{
APIConnString: s.APIConnString,
APIHost: s.APIHost,
APIPort: s.APIPort,
GRPCConnString: s.GRPCConnString,
GRPCHost: s.GRPCHost,
GRPCPort: s.GRPCPort,
GRPCSSL: s.GRPCSSL,
}
wgvals := models.WG{
GRPCWireGuard: w.GRPCWireGuard,
GRPCWGAddress: w.GRPCWGAddress,
GRPCWGPort: w.GRPCWGPort,
GRPCWGPubKey: w.GRPCWGPubKey,
GRPCWGEndpoint: s.APIHost,
}
accessToken.ServerConfig = servervals
accessToken.WG = wgvals
accessToken.ClientConfig.Network = netID
accessToken.ClientConfig.Key = accesskey.Value
accessToken.ClientConfig.LocalRange = privAddr
tokenjson, err := json.Marshal(accessToken)
if err != nil {
return accesskey, err
}
accesskey.AccessString = base64.StdEncoding.EncodeToString([]byte(tokenjson))
//validate accesskey
v := validator.New()
err = v.Struct(accesskey)
@ -716,10 +751,35 @@ func CreateAccessKey(accesskey models.AccessKey, network models.Network) (models
func GetSignupToken(netID string) (models.AccessKey, error) {
var accesskey models.AccessKey
address := net.JoinHostPort(servercfg.GetGRPCHost(), servercfg.GetGRPCPort())
var accessToken models.AccessToken
s := servercfg.GetServerConfig()
w := servercfg.GetWGConfig()
servervals := models.ServerConfig{
APIConnString: s.APIConnString,
APIHost: s.APIHost,
APIPort: s.APIPort,
GRPCConnString: s.GRPCConnString,
GRPCHost: s.GRPCHost,
GRPCPort: s.GRPCPort,
GRPCSSL: s.GRPCSSL,
}
wgvals := models.WG{
GRPCWireGuard: w.GRPCWireGuard,
GRPCWGAddress: w.GRPCWGAddress,
GRPCWGPort: w.GRPCWGPort,
GRPCWGPubKey: w.GRPCWGPubKey,
GRPCWGEndpoint: s.APIHost,
}
accessstringdec := address + "|" + netID + "|" + "" + "|"
accesskey.AccessString = base64.StdEncoding.EncodeToString([]byte(accessstringdec))
accessToken.ServerConfig = servervals
accessToken.WG = wgvals
tokenjson, err := json.Marshal(accessToken)
if err != nil {
return accesskey, err
}
accesskey.AccessString = base64.StdEncoding.EncodeToString([]byte(tokenjson))
return accesskey, nil
}
func getSignupToken(w http.ResponseWriter, r *http.Request) {

16
controllers/nodeHttpController.go
View file

@ -578,7 +578,7 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
}
var nodechange models.Node
nodechange.IsEgressGateway = true
nodechange.EgressGatewayRange = gateway.RangeString
nodechange.EgressGatewayRanges = gateway.Ranges
nodechange.PostUp = "iptables -A FORWARD -i " + node.Interface + " -j ACCEPT; iptables -t nat -A POSTROUTING -o " + gateway.Interface + " -j MASQUERADE"
nodechange.PostDown = "iptables -D FORWARD -i " + node.Interface + " -j ACCEPT; iptables -t nat -D POSTROUTING -o " + gateway.Interface + " -j MASQUERADE"
if gateway.PostUp != "" {
@ -612,7 +612,7 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
{"postup", nodechange.PostUp},
{"postdown", nodechange.PostDown},
{"isegressgateway", nodechange.IsEgressGateway},
{"egressgatewayrange", nodechange.EgressGatewayRange},
{"egressgatewayranges", nodechange.EgressGatewayRanges},
{"lastmodified", nodechange.LastModified},
}},
}
@ -636,10 +636,10 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
func ValidateEgressGateway(gateway models.EgressGatewayRequest) error {
var err error
isIp := functions.IsIpCIDR(gateway.RangeString)
empty := gateway.RangeString == ""
if empty || !isIp {
err = errors.New("IP Range Not Valid")
//isIp := functions.IsIpCIDR(gateway.RangeString)
empty := len(gateway.Ranges)==0
if empty {
err = errors.New("IP Ranges Cannot Be Empty")
}
empty = gateway.Interface == ""
if empty {
@ -670,7 +670,7 @@ func DeleteEgressGateway(network, macaddress string) (models.Node, error) {
}
nodechange.IsEgressGateway = false
nodechange.EgressGatewayRange = ""
nodechange.EgressGatewayRanges = []string{}
nodechange.PostUp = ""
nodechange.PostDown = ""
@ -685,7 +685,7 @@ func DeleteEgressGateway(network, macaddress string) (models.Node, error) {
{"postup", nodechange.PostUp},
{"postdown", nodechange.PostDown},
{"isegressgateway", nodechange.IsEgressGateway},
{"egressgatewayrange", nodechange.EgressGatewayRange},
{"egressgatewayranges", nodechange.EgressGatewayRanges},
{"lastmodified", nodechange.LastModified},
}},
}

21
controllers/nodeHttpController_test.go
View file

@ -34,7 +34,7 @@ func TestCheckIn(t *testing.T) {
func TestCreateEgressGateway(t *testing.T) {
var gateway models.EgressGatewayRequest
gateway.Interface = "eth0"
gateway.RangeString = "10.100.100.0/24"
gateway.Ranges = []string{"10.100.100.0/24"}
deleteNet(t)
createNet()
t.Run("NoNodes", func(t *testing.T) {
@ -62,18 +62,18 @@ func TestDeleteEgressGateway(t *testing.T) {
createTestNode(t)
testnode := createTestNode(t)
gateway.Interface = "eth0"
gateway.RangeString = "10.100.100.0/24"
gateway.Ranges = []string{"10.100.100.0/24"}
gateway.NetID = "skynet"
gateway.NodeID = testnode.MacAddress
t.Run("Success", func(t *testing.T) {
node, err := CreateEgressGateway(gateway)
assert.Nil(t, err)
assert.Equal(t, true, node.IsEgressGateway)
assert.Equal(t, "10.100.100.0/24", node.EgressGatewayRange)
assert.Equal(t, []string{"10.100.100.0/24"}, node.EgressGatewayRanges)
node, err = DeleteEgressGateway(gateway.NetID, gateway.NodeID)
assert.Nil(t, err)
assert.Equal(t, false, node.IsEgressGateway)
assert.Equal(t, "", node.EgressGatewayRange)
assert.Equal(t, "", node.EgressGatewayRanges)
assert.Equal(t, "", node.PostUp)
assert.Equal(t, "", node.PostDown)
})
@ -81,7 +81,7 @@ func TestDeleteEgressGateway(t *testing.T) {
node, err := DeleteEgressGateway(gateway.NetID, gateway.NodeID)
assert.Nil(t, err)
assert.Equal(t, false, node.IsEgressGateway)
assert.Equal(t, "", node.EgressGatewayRange)
assert.Equal(t, "", node.EgressGatewayRanges)
assert.Equal(t, "", node.PostUp)
assert.Equal(t, "", node.PostDown)
})
@ -162,16 +162,9 @@ func TestUncordonNode(t *testing.T) {
}
func TestValidateEgressGateway(t *testing.T) {
var gateway models.EgressGatewayRequest
t.Run("InvalidRange", func(t *testing.T) {
gateway.Interface = "eth0"
gateway.RangeString = "helloworld"
err := ValidateEgressGateway(gateway)
assert.NotNil(t, err)
assert.Equal(t, "IP Range Not Valid", err.Error())
})
t.Run("EmptyRange", func(t *testing.T) {
gateway.Interface = "eth0"
gateway.RangeString = ""
gateway.Ranges = []string{}
err := ValidateEgressGateway(gateway)
assert.NotNil(t, err)
assert.Equal(t, "IP Range Not Valid", err.Error())
@ -184,7 +177,7 @@ func TestValidateEgressGateway(t *testing.T) {
})
t.Run("Success", func(t *testing.T) {
gateway.Interface = "eth0"
gateway.RangeString = "10.100.100.0/24"
gateway.Ranges = []string{"10.100.100.0/24"}
err := ValidateEgressGateway(gateway)
assert.Nil(t, err)
})

2
functions/helpers.go
View file

@ -555,7 +555,7 @@ func GetNodeObj(id primitive.ObjectID) models.Node {
//Switch to REGEX?
func NameInNetworkCharSet(name string) bool {
charset := "abcdefghijklmnopqrstuvwxyz1234567890-_"
charset := "abcdefghijklmnopqrstuvwxyz1234567890-_."
for _, char := range name {
if !strings.Contains(charset, strings.ToLower(string(char))) {

1
go.mod
View file

@ -9,6 +9,7 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/jinzhu/copier v0.3.2 // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
github.com/stretchr/testify v1.6.1
github.com/txn2/txeh v1.3.0

2
go.sum
View file

@ -93,6 +93,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=

59
kube/mongo-statefulset.yaml Normal file
View file

@ -0,0 +1,59 @@
apiVersion: v1
kind: Service
metadata:
name: mongo
labels:
name: mongo
spec:
ports:
- port: 27017
targetPort: 27017
clusterIP: None
selector:
role: mongo
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mongo
spec:
serviceName: "mongo"
replicas: 1
selector:
matchLabels:
role: mongo
template:
metadata:
labels:
app: mongo
role: mongo
spec:
containers:
- name: mongo
image: mongo
env:
- name: MONGO_INITDB_ROOT_USERNAME
value: mongoadmin
- name: MONGO_INITDB_ROOT_PASSWORD
value: mongopass
securityContext:
privileged: true
volumeMounts:
- name: mongovol
mountPath: /data/db
volumes:
- name: mongovol
persistentVolumeClaim:
claimName: mongo-pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mongo-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 7Gi
storageClassName: longhorn

66
kube/netmaker-api.yaml Normal file
View file

@ -0,0 +1,66 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: netmaker
labels:
app: netmaker
spec:
selector:
matchLabels:
app: netmaker
replicas: 1
template:
metadata:
labels:
app: netmaker
spec:
containers:
- name: netmaker
image: gravitl/netmaker:v0.5
ports:
- containerPort: 8081
env:
- name: SERVER_HOST
value: "api.test.netmaker.org"
- name: API_PORT
value: "8081"
- name: AGENT_BACKEND
value: "off"
- name: CLIENT_MODE
value: "off"
- name: DNS_MODE
value: "off"
- name: MASTER_KEY
value: "Unkn0wn!"
- name: SERVER_GRPC_WIREGUARD
value: "off"
- name: MASTER_KEY
value: "secretkey"
- name: CORS_ALLOWED_ORIGIN
value: "*"
- name: DISABLE_REMOTE_IP_CHECK
value: "on"
- name: MONGO_ADMIN
value: "mongoadmin"
- name: MONGO_PASS
value: "mongopass"
- name: MONGO_HOST
value: "mongo-0.mongo"
- name: MONGO_OPTS
value: "/?authSource=admin"
---
apiVersion: v1
kind: Service
metadata:
labels:
app: netmaker
name: netmaker
spec:
ports:
- port: 8081
protocol: TCP
targetPort: 8081
selector:
app: netmaker
sessionAffinity: None
type: ClusterIP

68
kube/netmaker-grpc.yaml Normal file
View file

@ -0,0 +1,68 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: netmaker-grpc
labels:
app: netmaker-grpc
spec:
selector:
matchLabels:
app: netmaker-grpc
replicas: 1
template:
metadata:
labels:
app: netmaker-grpc
spec:
containers:
- name: netmaker-grpc
image: gravitl/netmaker:v0.5
ports:
- containerPort: 50051
env:
- name: SERVER_HOST
value: "grpc.test.netmaker.org"
- name: CLIENT_MODE
value: "off"
- name: DNS_MODE
value: "off"
- name: MASTER_KEY
value: "Unkn0wn!"
- name: SERVER_GRPC_WIREGUARD
value: "off"
- name: MASTER_KEY
value: "secretkey"
- name: CORS_ALLOWED_ORIGIN
value: "*"
- name: DISABLE_REMOTE_IP_CHECK
value: "on"
- name: MONGO_ADMIN
value: "mongoadmin"
- name: MONGO_PASS
value: "mongopass"
- name: MONGO_HOST
value: "mongo-0.mongo"
- name: MONGO_OPTS
value: "/?authSource=admin"
- name: SERVER_GRPC_HOST
value: "0.0.0.0"
- name: GRPC_PORT
value: "50051"
- name: REST_BACKEND
value: "off"
---
apiVersion: v1
kind: Service
metadata:
labels:
app: netmaker-grpc
name: netmaker-grpc
spec:
ports:
- port: 50051
protocol: TCP
targetPort: 50051
selector:
app: netmaker-grpc
sessionAffinity: None
type: ClusterIP

25
kube/netmaker-ingress-api.yaml Normal file
View file

@ -0,0 +1,25 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "traefik"
kubernetes.io/ingress.allow-http: "false"
traefik.ingress.kubernetes.io/redirect-entry-point: https
traefik.ingress.kubernetes.io/redirect-permanent: "true"
traefik.ingress.kubernetes.io/rule-type: "PathPrefixStrip"
cert-manager.io/cluster-issuer: wildcard-issuer
name: nm-api-ingress
namespace: netmaker
spec:
rules:
- host: api.test.netmaker.org
http:
paths:
- path: /
backend:
serviceName: netmaker
servicePort: 8081
tls:
- hosts:
- api.test.netmaker.org
secretName: cert-nm-api

25
kube/netmaker-ingress-frontend.yaml Normal file
View file

@ -0,0 +1,25 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "traefik"
kubernetes.io/ingress.allow-http: "false"
traefik.ingress.kubernetes.io/redirect-entry-point: https
traefik.ingress.kubernetes.io/redirect-permanent: "true"
traefik.ingress.kubernetes.io/rule-type: "PathPrefixStrip"
cert-manager.io/cluster-issuer: wildcard-issuer
name: nm-ui-ingress
namespace: netmaker
spec:
rules:
- host: nm.test.netmaker.org
http:
paths:
- path: /
backend:
serviceName: netmaker-ui
servicePort: 80
tls:
- hosts:
- nm.test.netmaker.org
secretName: cert-nm-ui

17
kube/netmaker-ingress-grpc.yaml Normal file
View file

@ -0,0 +1,17 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "traefik"
ingress.kubernetes.io/protocol: "h2c"
name: nm-grpc-ingress
namespace: netmaker
spec:
rules:
- host: grpc.test.netmaker.org
http:
paths:
- path: /
backend:
serviceName: netmaker-grpc
servicePort: 50051

40
kube/netmaker-ui.yaml Normal file
View file

@ -0,0 +1,40 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: netmaker-ui
labels:
app: netmaker-ui
spec:
selector:
matchLabels:
app: netmaker-ui
replicas: 1
template:
metadata:
labels:
app: netmaker-ui
spec:
containers:
- name: netmaker-ui
image: gravitl/netmaker-ui:v0.5
ports:
- containerPort: 80
env:
- name: BACKEND_URL
value: "https://api.test.netmaker.org"
---
apiVersion: v1
kind: Service
metadata:
labels:
app: netmaker-ui
name: netmaker-ui
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: netmaker-ui
sessionAffinity: None
type: ClusterIP

9
main.go
View file

@ -146,9 +146,12 @@ func runGRPC(wg *sync.WaitGroup, installserver bool) {
log.Println("Agent Server succesfully started on port " + grpcport + " (gRPC)")
if installserver {
log.Println("Adding server to default network")
success, err := serverctl.AddNetwork("default")
if err != nil {
success := true
if !servercfg.DisableDefaultNet() {
log.Println("Adding server to default network")
success, err = serverctl.AddNetwork("default")
}
if err != nil {
log.Printf("Error adding to default network: %v", err)
log.Println("Unable to add server to network. Continuing.")
log.Println("Please investigate client installation on server.")

31
models/accessToken.go Normal file
View file

@ -0,0 +1,31 @@
package models
type AccessToken struct {
ServerConfig
ClientConfig
WG
}
type ClientConfig struct {
Network string `json:"network"`
Key string `json:"key"`
LocalRange string `json:"localrange"`
}
type ServerConfig struct {
APIConnString string `json:"apiconn"`
APIHost string `json:"apihost"`
APIPort string `json:"apiport"`
GRPCConnString string `json:"grpcconn"`
GRPCHost string `json:"grpchost"`
GRPCPort string `json:"grpcport"`
GRPCSSL string `json:"grpcssl"`
}
type WG struct {
GRPCWireGuard string `json:"grpcwg"`
GRPCWGAddress string `json:"grpcwgaddr"`
GRPCWGPort string `json:"grpcwgport"`
GRPCWGPubKey string `json:"grpcwgpubkey"`
GRPCWGEndpoint string `json:"grpcwgendpoint"`
}

14
models/network.go
View file

@ -16,8 +16,8 @@ type Network struct {
// AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"required_with=isdualstack true,cidrv6"`
AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"addressrange6_valid"`
//can't have min=1 with omitempty
DisplayName string `json:"displayname,omitempty" bson:"displayname,omitempty" validate:"omitempty,alphanum,min=2,max=20,displayname_unique"`
NetID string `json:"netid" bson:"netid" validate:"required,alphanum,min=1,max=12,netid_valid"`
DisplayName string `json:"displayname,omitempty" bson:"displayname,omitempty" validate:"omitempty,min=1,max=20,displayname_valid"`
NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=12,netid_valid"`
NodesLastModified int64 `json:"nodeslastmodified" bson:"nodeslastmodified"`
NetworkLastModified int64 `json:"networklastmodified" bson:"networklastmodified"`
DefaultInterface string `json:"defaultinterface" bson:"defaultinterface"`
@ -47,8 +47,8 @@ type NetworkUpdate struct {
// AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"required_with=isdualstack true,cidrv6"`
AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"omitempty,cidr"`
//can't have min=1 with omitempty
DisplayName string `json:"displayname,omitempty" bson:"displayname,omitempty" validate:"omitempty,alphanum,min=2,max=20"`
NetID string `json:"netid" bson:"netid" validate:"omitempty,alphanum,min=1,max=12"`
DisplayName string `json:"displayname,omitempty" bson:"displayname,omitempty" validate:"omitempty,netid_valid,min=1,max=20"`
NetID string `json:"netid" bson:"netid" validate:"omitempty,netid_valid,min=1,max=15"`
NodesLastModified int64 `json:"nodeslastmodified" bson:"nodeslastmodified"`
NetworkLastModified int64 `json:"networklastmodified" bson:"networklastmodified"`
DefaultInterface string `json:"defaultinterface" bson:"defaultinterface"`
@ -86,7 +86,11 @@ func (network *Network) SetDefaults() {
network.DisplayName = network.NetID
}
if network.DefaultInterface == "" {
network.DefaultInterface = "nm-" + network.NetID
if len(network.NetID) < 13 {
network.DefaultInterface = "nm-" + network.NetID
} else {
network.DefaultInterface = network.NetID
}
}
if network.DefaultListenPort == 0 {
network.DefaultListenPort = 51821

19
models/node.go
View file

@ -28,7 +28,7 @@ type Node struct {
Endpoint string `json:"endpoint" bson:"endpoint" validate:"required,ip"`
PostUp string `json:"postup" bson:"postup"`
PostDown string `json:"postdown" bson:"postdown"`
AllowedIPs string `json:"allowedips" bson:"allowedips"`
AllowedIPs []string `json:"allowedips" bson:"allowedips"`
PersistentKeepalive int32 `json:"persistentkeepalive" bson:"persistentkeepalive" validate:"omitempty,numeric,max=1000"`
SaveConfig *bool `json:"saveconfig" bson:"saveconfig"`
AccessKey string `json:"accesskey" bson:"accesskey"`
@ -45,9 +45,11 @@ type Node struct {
IsPending bool `json:"ispending" bson:"ispending"`
IsEgressGateway bool `json:"isegressgateway" bson:"isegressgateway"`
IsIngressGateway bool `json:"isingressgateway" bson:"isingressgateway"`
EgressGatewayRange string `json:"egressgatewayrange" bson:"egressgatewayrange"`
EgressGatewayRanges []string `json:"egressgatewayranges" bson:"egressgatewayranges"`
IngressGatewayRange string `json:"ingressgatewayrange" bson:"ingressgatewayrange"`
PostChanges string `json:"postchanges" bson:"postchanges"`
StaticIP string `json:"staticip" bson:"staticip"`
StaticPubKey string `json:"staticpubkey" bson:"staticpubkey"`
}
//node update struct --- only validations are different
@ -62,7 +64,7 @@ type NodeUpdate struct {
Endpoint string `json:"endpoint" bson:"endpoint" validate:"omitempty,ip"`
PostUp string `json:"postup" bson:"postup"`
PostDown string `json:"postdown" bson:"postdown"`
AllowedIPs string `json:"allowedips" bson:"allowedips"`
AllowedIPs []string `json:"allowedips" bson:"allowedips"`
PersistentKeepalive int32 `json:"persistentkeepalive" bson:"persistentkeepalive" validate:"omitempty,numeric,max=1000"`
SaveConfig *bool `json:"saveconfig" bson:"saveconfig"`
AccessKey string `json:"accesskey" bson:"accesskey"`
@ -80,8 +82,10 @@ type NodeUpdate struct {
IsIngressGateway bool `json:"isingressgateway" bson:"isingressgateway"`
IsEgressGateway bool `json:"isegressgateway" bson:"isegressgateway"`
IngressGatewayRange string `json:"ingressgatewayrange" bson:"ingressgatewayrange"`
EgressGatewayRange string `json:"gatewayrange" bson:"gatewayrange"`
EgressGatewayRanges []string `json:"egressgatewayranges" bson:"egressgatewayranges"`
PostChanges string `json:"postchanges" bson:"postchanges"`
StaticIP string `json:"staticip" bson:"staticip"`
StaticPubKey string `json:"staticpubkey" bson:"staticpubkey"`
}
//Duplicated function for NodeUpdates
@ -191,6 +195,13 @@ func (node *Node) SetDefaults() {
postup := parentNetwork.DefaultPostUp
node.PostUp = postup
}
if node.StaticIP == "" {
node.StaticIP = "no"
}
if node.StaticPubKey == "" {
node.StaticPubKey = "no"
}
node.CheckInInterval = parentNetwork.DefaultCheckInInterval
}

95
netclient/config/config.go
View file

@ -6,15 +6,15 @@ import (
"os"
"encoding/base64"
"errors"
"strings"
"fmt"
"net"
"log"
"encoding/json"
"gopkg.in/yaml.v3"
nodepb "github.com/gravitl/netmaker/grpc"
"github.com/gravitl/netmaker/models"
)
type GlobalConfig struct {
GRPCWireGuard string `yaml:"grpcwg"`
Client models.IntClient
}
@ -29,6 +29,8 @@ type ServerConfig struct {
GRPCAddress string `yaml:"grpcaddress"`
APIAddress string `yaml:"apiaddress"`
AccessKey string `yaml:"accesskey"`
GRPCSSL string `yaml:"grpcssl"`
GRPCWireGuard string `yaml:"grpcwg"`
}
type ListConfig struct {
@ -53,16 +55,19 @@ type NodeConfig struct {
IsLocal string `yaml:"islocal"`
IsDualStack string `yaml:"isdualstack"`
IsIngressGateway string `yaml:"isingressgateway"`
AllowedIPs string `yaml:"allowedips"`
AllowedIPs []string `yaml:"allowedips"`
LocalRange string `yaml:"localrange"`
PostUp string `yaml:"postup"`
PostDown string `yaml:"postdown"`
Port int32 `yaml:"port"`
KeepAlive int32 `yaml:"keepalive"`
PublicKey string `yaml:"publickey"`
ServerPubKey string `yaml:"serverpubkey"`
PrivateKey string `yaml:"privatekey"`
Endpoint string `yaml:"endpoint"`
PostChanges string `yaml:"postchanges"`
StaticIP string `yaml:"staticip"`
StaticPubKey string `yaml:"staticpubkey"`
IPForwarding string `yaml:"ipforwarding"`
}
@ -375,16 +380,34 @@ func GetCLIConfig(c *cli.Context) (ClientConfig, error){
log.Println("error decoding token")
return cfg, err
}
token := string(tokenbytes)
tokenvals := strings.Split(token, "|")
cfg.Server.GRPCAddress = tokenvals[1]
cfg.Network = tokenvals[3]
cfg.Node.Network = tokenvals[3]
cfg.Server.AccessKey = tokenvals[4]
if len(tokenvals) > 4 {
cfg.Node.LocalRange = tokenvals[5]
var accesstoken models.AccessToken
if err := json.Unmarshal(tokenbytes, &accesstoken); err != nil {
log.Println("error converting token json to object", tokenbytes )
return cfg, err
}
if accesstoken.ServerConfig.APIConnString != "" {
cfg.Server.APIAddress = accesstoken.ServerConfig.APIConnString
} else {
cfg.Server.APIAddress = accesstoken.ServerConfig.APIHost
if accesstoken.ServerConfig.APIPort != "" {
cfg.Server.APIAddress = cfg.Server.APIAddress + ":" + accesstoken.ServerConfig.APIPort
}
}
if accesstoken.ServerConfig.GRPCConnString != "" {
cfg.Server.GRPCAddress = accesstoken.ServerConfig.GRPCConnString
} else {
cfg.Server.GRPCAddress = accesstoken.ServerConfig.GRPCHost
if accesstoken.ServerConfig.GRPCPort != "" {
cfg.Server.GRPCAddress = cfg.Server.GRPCAddress + ":" + accesstoken.ServerConfig.GRPCPort
}
}
cfg.Network = accesstoken.ClientConfig.Network
cfg.Node.Network = accesstoken.ClientConfig.Network
cfg.Server.AccessKey = accesstoken.ClientConfig.Key
cfg.Node.LocalRange = accesstoken.ClientConfig.LocalRange
cfg.Server.GRPCSSL = accesstoken.ServerConfig.GRPCSSL
cfg.Server.GRPCWireGuard = accesstoken.WG.GRPCWireGuard
if c.String("grpcserver") != "" {
cfg.Server.GRPCAddress = c.String("grpcserver")
}
@ -401,6 +424,13 @@ func GetCLIConfig(c *cli.Context) (ClientConfig, error){
if c.String("localrange") != "" {
cfg.Node.LocalRange = c.String("localrange")
}
if c.String("grpcssl") != "" {
cfg.Server.GRPCSSL = c.String("grpcssl")
}
if c.String("grpcwg") != "" {
cfg.Server.GRPCWireGuard = c.String("grpcwg")
}
} else {
cfg.Server.GRPCAddress = c.String("grpcserver")
cfg.Server.APIAddress = c.String("apiserver")
@ -408,6 +438,8 @@ func GetCLIConfig(c *cli.Context) (ClientConfig, error){
cfg.Network = c.String("network")
cfg.Node.Network = c.String("network")
cfg.Node.LocalRange = c.String("localrange")
cfg.Server.GRPCWireGuard = c.String("grpcwg")
cfg.Server.GRPCSSL = c.String("grpcssl")
}
cfg.Node.Name = c.String("name")
cfg.Node.Interface = c.String("interface")
@ -442,22 +474,33 @@ func GetCLIConfigRegister(c *cli.Context) (GlobalConfig, error){
log.Println("error decoding token")
return cfg, err
}
token := string(tokenbytes)
tokenvals := strings.Split(token, "|")
cfg.Client.ServerPrivateAddress, cfg.Client.ServerGRPCPort, err = net.SplitHostPort(tokenvals[1])
var accesstoken models.AccessToken
if err := json.Unmarshal(tokenbytes, &accesstoken); err != nil {
log.Println("error converting token json to object", tokenbytes )
return cfg, err
}
cfg.GRPCWireGuard = accesstoken.WG.GRPCWireGuard
cfg.Client.ServerPrivateAddress = accesstoken.WG.GRPCWGAddress
cfg.Client.ServerGRPCPort = accesstoken.WG.GRPCWGPort
if err != nil {
log.Println("error decoding token grpcserver")
return cfg, err
}
cfg.Client.ServerPublicEndpoint, cfg.Client.ServerAPIPort, err = net.SplitHostPort(tokenvals[2])
if err != nil {
log.Println("error decoding token apiserver")
return cfg, err
}
cfg.Client.ServerWGPort = tokenvals[0]
cfg.Client.ServerKey = tokenvals[4]
if err != nil {
log.Println("error decoding token apiserver")
return cfg, err
}
if accesstoken.ServerConfig.APIConnString != "" {
cfg.Client.ServerPublicEndpoint = accesstoken.ServerConfig.APIConnString
} else {
cfg.Client.ServerPublicEndpoint = accesstoken.ServerConfig.APIHost
if accesstoken.ServerConfig.APIPort != "" {
cfg.Client.ServerAPIPort = accesstoken.ServerConfig.APIPort
}
}
cfg.Client.ServerWGPort = accesstoken.WG.GRPCWGPort
cfg.Client.ServerKey = accesstoken.ClientConfig.Key
cfg.Client.ServerKey = accesstoken.WG.GRPCWGPubKey
if c.String("grpcserver") != "" {
cfg.Client.ServerPrivateAddress = c.String("grpcserver")
@ -465,8 +508,8 @@ func GetCLIConfigRegister(c *cli.Context) (GlobalConfig, error){
if c.String("apiserver") != "" {
cfg.Client.ServerPublicEndpoint = c.String("apiserver")
}
if c.String("key") != "" {
cfg.Client.ServerKey = c.String("key")
if c.String("pubkey") != "" {
cfg.Client.ServerKey = c.String("pubkey")
}
if c.String("network") != "all" {
cfg.Client.Network = c.String("network")

15
netclient/functions/checkin.go
View file

@ -1,6 +1,8 @@
package functions
import (
"google.golang.org/grpc/credentials"
"crypto/tls"
"fmt"
"context"
"strings"
@ -120,10 +122,13 @@ func CheckIn(network string) error {
nodecfg = cfg.Node
}
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(servercfg.GRPCAddress, requestOpts)
if err != nil {
fmt.Printf("Cant dial GRPC server: %v", err)
@ -296,6 +301,10 @@ func Pull (network string) error{
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(servercfg.GRPCAddress, requestOpts)
if err != nil {
fmt.Printf("Cant dial GRPC server: %v", err)
@ -342,6 +351,10 @@ func Push (network string) error{
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(servercfg.GRPCAddress, requestOpts)
if err != nil {
fmt.Printf("Cant dial GRPC server: %v", err)

6
netclient/functions/common.go
View file

@ -1,6 +1,8 @@
package functions
import (
"google.golang.org/grpc/credentials"
"crypto/tls"
"fmt"
"encoding/json"
"errors"
@ -287,6 +289,10 @@ func LeaveNetwork(network string) error {
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(servercfg.GRPCAddress, requestOpts)
if err != nil {
log.Printf("Unable to establish client connection to " + servercfg.GRPCAddress + ": %v", err)

18
netclient/functions/join.go
View file

@ -1,6 +1,8 @@
package functions
import (
"google.golang.org/grpc/credentials"
"crypto/tls"
"fmt"
"errors"
"context"
@ -26,7 +28,7 @@ func JoinNetwork(cfg config.ClientConfig) error {
err := errors.New("ALREADY_INSTALLED. Netclient appears to already be installed for " + cfg.Network + ". To re-install, please remove by executing 'sudo netclient leave -n " + cfg.Network + "'. Then re-run the install command.")
return err
}
log.Println("attempting to joining " + cfg.Network + " at " + cfg.Server.GRPCAddress)
log.Println("attempting to join " + cfg.Network + " at " + cfg.Server.GRPCAddress)
err := config.Write(&cfg, cfg.Network)
if err != nil {
return err
@ -139,11 +141,17 @@ func JoinNetwork(cfg config.ClientConfig) error {
}
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
conn, err := grpc.Dial(cfg.Server.GRPCAddress, requestOpts)
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(cfg.Server.GRPCAddress, requestOpts)
if err != nil {
log.Fatalf("Unable to establish client connection to localhost:50051: %v", err)
log.Fatalf("Unable to establish client connection to " + cfg.Server.GRPCAddress + ": %v", err)
}
wcclient = nodepb.NewNodeServiceClient(conn)
postnode := &nodepb.Node{
@ -165,6 +173,7 @@ func JoinNetwork(cfg config.ClientConfig) error {
if err != nil {
return err
}
res, err := wcclient.CreateNode(
context.TODO(),
&nodepb.CreateNodeReq{
@ -189,7 +198,6 @@ func JoinNetwork(cfg config.ClientConfig) error {
}
node.Endpoint = node.Localaddress
}
err = config.ModConfig(node)
if err != nil {
return err

6
netclient/main.go
View file

@ -200,7 +200,11 @@ func main() {
Flags: cliFlags,
Action: func(c *cli.Context) error {
cfg, err := config.GetCLIConfigRegister(c)
if err != nil {
if err != nil {
return err
}
if cfg.GRPCWireGuard == "off" {
log.Println("Server is not using WireGuard to secure GRPC. Skipping.")
return err
}
if cfg.Client.ServerPrivateAddress == "" {

7
netclient/server/grpc.go
View file

@ -1,6 +1,8 @@
package server
import (
"google.golang.org/grpc/credentials"
"crypto/tls"
"fmt"
"context"
"log"
@ -73,10 +75,13 @@ func RemoveNetwork(network string) error {
node := cfg.Node
fmt.Println("Deleting remote node with MAC: " + node.MacAddress)
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(servercfg.GRPCAddress, requestOpts)
if err != nil {
log.Printf("Unable to establish client connection to " + servercfg.GRPCAddress + ": %v", err)

14
netclient/wireguard/kernel.go
View file

@ -2,6 +2,8 @@ package wireguard
import (
//"github.com/davecgh/go-spew/spew"
"google.golang.org/grpc/credentials"
"crypto/tls"
"fmt"
"strconv"
"errors"
@ -328,10 +330,20 @@ func SetWGKeyConfig(network string, serveraddr string) error {
ctx := context.Background()
var header metadata.MD
cfg, err := config.ReadConfig(network)
if err != nil {
return err
}
var wcclient nodepb.NodeServiceClient
var requestOpts grpc.DialOption
requestOpts = grpc.WithInsecure()
conn, err := grpc.Dial(serveraddr, requestOpts)
if cfg.Server.GRPCSSL == "on" {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
requestOpts = grpc.WithTransportCredentials(h2creds)
}
conn, err := grpc.Dial(serveraddr, requestOpts)
if err != nil {
fmt.Printf("Cant dial GRPC server: %v", err)
return err

57
servercfg/serverconf.go
View file

@ -19,8 +19,10 @@ func SetHost() error {
}
func GetServerConfig() config.ServerConfig {
var cfg config.ServerConfig
cfg.APIConnString = GetAPIConnString()
cfg.APIHost = GetAPIHost()
cfg.APIPort = GetAPIPort()
cfg.GRPCConnString = GetGRPCConnString()
cfg.GRPCHost = GetGRPCHost()
cfg.GRPCPort = GetGRPCPort()
cfg.MasterKey = "(hidden)"
@ -41,10 +43,18 @@ func GetServerConfig() config.ServerConfig {
if IsDNSMode() {
cfg.DNSMode = "on"
}
cfg.GRPCSSL = "off"
if IsGRPCSSL() {
cfg.GRPCSSL = "on"
}
cfg.DisableRemoteIPCheck = "off"
if DisableRemoteIPCheck() {
cfg.DisableRemoteIPCheck = "on"
}
cfg.DisableDefaultNet = "off"
if DisableDefaultNet() {
cfg.DisableRemoteIPCheck = "on"
}
return cfg
}
@ -63,7 +73,15 @@ func GetWGConfig() config.WG{
cfg.GRPCWGPrivKey = GetGRPCWGPrivKey()
return cfg
}
func GetAPIConnString() string {
conn := ""
if os.Getenv("SERVER_API_CONN_STRING") != "" {
conn = os.Getenv("SERVER_API_CONN_STRING")
} else if config.Config.Server.APIConnString != "" {
conn = config.Config.Server.APIConnString
}
return conn
}
func GetAPIHost() string {
serverhost := "127.0.0.1"
if os.Getenv("SERVER_HTTP_HOST") != "" {
@ -101,6 +119,15 @@ func GetDefaultNodeLimit() int32 {
}
return limit
}
func GetGRPCConnString() string {
conn := ""
if os.Getenv("SERVER_GRPC_CONN_STRING") != "" {
conn = os.Getenv("SERVER_GRPC_CONN_STRING")
} else if config.Config.Server.GRPCConnString != "" {
conn = config.Config.Server.GRPCConnString
}
return conn
}
func GetGRPCHost() string {
serverhost := "127.0.0.1"
@ -201,6 +228,21 @@ func IsDNSMode() bool {
}
return isdns
}
func IsGRPCSSL() bool {
isssl := false
if os.Getenv("GRPC_SSL") != "" {
if os.Getenv("GRPC_SSL") == "on" {
isssl = true
}
} else if config.Config.Server.DNSMode != "" {
if config.Config.Server.DNSMode == "on" {
isssl = true
}
}
return isssl
}
func DisableRemoteIPCheck() bool {
disabled := false
if os.Getenv("DISABLE_REMOTE_IP_CHECK") != "" {
@ -214,6 +256,19 @@ func DisableRemoteIPCheck() bool {
}
return disabled
}
func DisableDefaultNet() bool {
disabled := false
if os.Getenv("DISABLE_DEFAULT_NET") != "" {
if os.Getenv("DISABLE_DEFAULT_NET") == "on" {
disabled = true
}
} else if config.Config.Server.DisableDefaultNet != "" {
if config.Config.Server.DisableDefaultNet == "on" {
disabled= true
}
}
return disabled
}
func GetPublicIP() (string, error) {
endpoint := ""

17
test/node_test.go
View file

@ -274,7 +274,7 @@ func TestCreateEgressGateway(t *testing.T) {
//assert.False(t, node.IsEgressGateway/g)
var gateway models.EgressGatewayRequest
t.Run("Valid", func(t *testing.T) {
gateway.RangeString = "0.0.0.0/0"
gateway.Ranges = []string{"0.0.0.0/0"}
gateway.Interface = "eth0"
response, err := api(t, gateway, http.MethodPost, baseURL+"/api/nodes/skynet/01:02:03:04:05:06/creategateway", "secretkey")
assert.Nil(t, err, err)
@ -286,21 +286,8 @@ func TestCreateEgressGateway(t *testing.T) {
assert.True(t, message.IsEgressGateway)
t.Log(err)
})
t.Run("BadRange", func(t *testing.T) {
gateway.RangeString = "0.0.0.0/36"
gateway.Interface = "eth0"
response, err := api(t, gateway, http.MethodPost, baseURL+"/api/nodes/skynet/01:02:03:04:05:06/creategateway", "secretkey")
assert.Nil(t, err, err)
assert.Equal(t, http.StatusInternalServerError, response.StatusCode)
defer response.Body.Close()
var message models.ErrorResponse
err = json.NewDecoder(response.Body).Decode(&message)
assert.Nil(t, err, err)
assert.Equal(t, http.StatusInternalServerError, message.Code)
assert.Equal(t, "IP Range Not Valid", message.Message)
})
t.Run("BadInterface", func(t *testing.T) {
gateway.RangeString = "0.0.0.0/0"
gateway.Ranges = []string{"0.0.0.0/0"}
gateway.Interface = ""
response, err := api(t, gateway, http.MethodPost, baseURL+"/api/nodes/skynet/01:02:03:04:05:06/creategateway", "secretkey")
assert.Nil(t, err, err)