From 7f40371ffdf6191b0b0e03e51bbdecb6119e2029 Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi <51291657+VishalDalwadi@users.noreply.github.com> Date: Sat, 12 Apr 2025 03:07:57 -0700 Subject: [PATCH] Task/DB-Migration: Add Key-Value to SQL Migration functionality. (#3380) * feat(go): add db package; * feat(go): add jobs table; * feat(go): add schema migration facade; * refactor(go): use custom key type to avoid collisions; --- database/database.go | 58 ++++++------ db/connector.go | 28 ++++++ db/db.go | 99 +++++++++++++++++++++ db/postgres.go | 49 ++++++++++ db/sqlite.go | 54 +++++++++++ go.mod | 21 +++-- go.sum | 28 ++++++ migrate/migrate_schema.go | 183 ++++++++++++++++++++++++++++++++++++++ schema/jobs.go | 36 ++++++++ schema/models.go | 8 ++ 10 files changed, 533 insertions(+), 31 deletions(-) create mode 100644 db/connector.go create mode 100644 db/db.go create mode 100644 db/postgres.go create mode 100644 db/sqlite.go create mode 100644 migrate/migrate_schema.go create mode 100644 schema/jobs.go create mode 100644 schema/models.go diff --git a/database/database.go b/database/database.go index c17465ba..b1080c8a 100644 --- a/database/database.go +++ b/database/database.go @@ -102,6 +102,35 @@ const ( var dbMutex sync.RWMutex +var Tables = []string{ + NETWORKS_TABLE_NAME, + NODES_TABLE_NAME, + CERTS_TABLE_NAME, + DELETED_NODES_TABLE_NAME, + USERS_TABLE_NAME, + DNS_TABLE_NAME, + EXT_CLIENT_TABLE_NAME, + PEERS_TABLE_NAME, + SERVERCONF_TABLE_NAME, + SERVER_UUID_TABLE_NAME, + GENERATED_TABLE_NAME, + NODE_ACLS_TABLE_NAME, + SSO_STATE_CACHE, + METRICS_TABLE_NAME, + NETWORK_USER_TABLE_NAME, + USER_GROUPS_TABLE_NAME, + CACHE_TABLE_NAME, + HOSTS_TABLE_NAME, + ENROLLMENT_KEYS_TABLE_NAME, + HOST_ACTIONS_TABLE_NAME, + PENDING_USERS_TABLE_NAME, + USER_PERMISSIONS_TABLE_NAME, + USER_INVITES_TABLE_NAME, + TAG_TABLE_NAME, + ACLS_TABLE_NAME, + PEER_ACK_TABLE, +} + func getCurrentDB() map[string]interface{} { switch servercfg.GetDB() { case "rqlite": @@ -135,32 +164,9 @@ func InitializeDatabase() error { } func createTables() { - CreateTable(NETWORKS_TABLE_NAME) - CreateTable(NODES_TABLE_NAME) - CreateTable(CERTS_TABLE_NAME) - CreateTable(DELETED_NODES_TABLE_NAME) - CreateTable(USERS_TABLE_NAME) - CreateTable(DNS_TABLE_NAME) - CreateTable(EXT_CLIENT_TABLE_NAME) - CreateTable(PEERS_TABLE_NAME) - CreateTable(SERVERCONF_TABLE_NAME) - CreateTable(SERVER_UUID_TABLE_NAME) - CreateTable(GENERATED_TABLE_NAME) - CreateTable(NODE_ACLS_TABLE_NAME) - CreateTable(SSO_STATE_CACHE) - CreateTable(METRICS_TABLE_NAME) - CreateTable(NETWORK_USER_TABLE_NAME) - CreateTable(USER_GROUPS_TABLE_NAME) - CreateTable(CACHE_TABLE_NAME) - CreateTable(HOSTS_TABLE_NAME) - CreateTable(ENROLLMENT_KEYS_TABLE_NAME) - CreateTable(HOST_ACTIONS_TABLE_NAME) - CreateTable(PENDING_USERS_TABLE_NAME) - CreateTable(USER_PERMISSIONS_TABLE_NAME) - CreateTable(USER_INVITES_TABLE_NAME) - CreateTable(TAG_TABLE_NAME) - CreateTable(ACLS_TABLE_NAME) - CreateTable(PEER_ACK_TABLE) + for _, table := range Tables { + _ = CreateTable(table) + } } func CreateTable(tableName string) error { diff --git a/db/connector.go b/db/connector.go new file mode 100644 index 00000000..00448b7e --- /dev/null +++ b/db/connector.go @@ -0,0 +1,28 @@ +package db + +import ( + "errors" + "github.com/gravitl/netmaker/servercfg" + "gorm.io/gorm" +) + +var ErrUnsupportedDB = errors.New("unsupported db type") + +// connector helps connect to a database, +// along with any initializations required. +type connector interface { + connect() (*gorm.DB, error) +} + +// newConnector detects the database being +// used and returns the corresponding connector. +func newConnector() (connector, error) { + switch servercfg.GetDB() { + case "sqlite": + return &sqliteConnector{}, nil + case "postgres": + return &postgresConnector{}, nil + default: + return nil, ErrUnsupportedDB + } +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 00000000..2f9f82a6 --- /dev/null +++ b/db/db.go @@ -0,0 +1,99 @@ +package db + +import ( + "context" + "errors" + "gorm.io/gorm" + "net/http" + "time" +) + +type ctxKey string + +const dbCtxKey ctxKey = "db" + +var db *gorm.DB + +var ErrDBNotFound = errors.New("no db instance in context") + +// InitializeDB initializes a connection to the +// database (if not already done) and ensures it +// has the latest schema. +func InitializeDB(models ...interface{}) error { + if db != nil { + return nil + } + + connector, err := newConnector() + if err != nil { + return err + } + + // DB / LIFE ADVICE: try 5 times before giving up. + for i := 0; i < 5; i++ { + db, err = connector.connect() + if err == nil { + break + } + + // wait 2s if you have the time. + time.Sleep(2 * time.Second) + } + if err != nil { + return err + } + + return db.AutoMigrate(models...) +} + +// WithContext returns a new context with the db +// connection instance. +// +// Ensure InitializeDB has been called before using +// this function. +// +// To extract the db connection use the FromContext +// function. +func WithContext(ctx context.Context) context.Context { + return context.WithValue(ctx, dbCtxKey, db) +} + +// Middleware to auto-inject the db connection instance +// in a request's context. +// +// Ensure InitializeDB has been called before using this +// middleware. +func Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(WithContext(r.Context()))) + }) +} + +// FromContext extracts the db connection instance from +// the given context. +// +// The function panics, if a connection does not exist. +func FromContext(ctx context.Context) *gorm.DB { + db, ok := ctx.Value(dbCtxKey).(*gorm.DB) + if !ok { + panic(ErrDBNotFound) + } + + return db +} + +// BeginTx returns a context with a new transaction. +// If the context already has a db connection instance, +// it uses that instance. Otherwise, it uses the +// connection initialized in the package. +// +// Ensure InitializeDB has been called before using +// this function. +func BeginTx(ctx context.Context) context.Context { + dbInCtx, ok := ctx.Value(dbCtxKey).(*gorm.DB) + if !ok { + return context.WithValue(ctx, dbCtxKey, db.Begin()) + } + + return context.WithValue(ctx, dbCtxKey, dbInCtx.Begin()) +} diff --git a/db/postgres.go b/db/postgres.go new file mode 100644 index 00000000..9f56db18 --- /dev/null +++ b/db/postgres.go @@ -0,0 +1,49 @@ +package db + +import ( + "fmt" + "github.com/gravitl/netmaker/servercfg" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// postgresConnector for initializing and +// connecting to a postgres database. +type postgresConnector struct{} + +// postgresConnector.connect connects and +// initializes a connection to postgres. +func (pg *postgresConnector) connect() (*gorm.DB, error) { + pgConf := servercfg.GetSQLConf() + dsn := fmt.Sprintf( + "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=5", + pgConf.Host, + pgConf.Port, + pgConf.Username, + pgConf.Password, + pgConf.DB, + pgConf.SSLMode, + ) + + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, err + } + + // ensure netmaker_v1 schema exists. + err = db.Exec("CREATE SCHEMA IF NOT EXISTS netmaker_v1").Error + if err != nil { + return nil, err + } + + // set the netmaker_v1 schema as the default schema. + err = db.Exec("SET search_path TO netmaker_v1").Error + if err != nil { + return nil, err + } + + return db, nil +} diff --git a/db/sqlite.go b/db/sqlite.go new file mode 100644 index 00000000..950123fa --- /dev/null +++ b/db/sqlite.go @@ -0,0 +1,54 @@ +package db + +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "os" + "path/filepath" +) + +// sqliteConnector for initializing and +// connecting to a sqlite database. +type sqliteConnector struct{} + +// sqliteConnector.connect connects and +// initializes a connection to sqlite. +func (s *sqliteConnector) connect() (*gorm.DB, error) { + // ensure data dir exists. + _, err := os.Stat("data") + if err != nil { + if os.IsNotExist(err) { + err = os.Mkdir("data", 0700) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + dbFilePath := filepath.Join("data", "netmaker_v1.db") + + // ensure netmaker_v1.db exists. + _, err = os.Stat(dbFilePath) + if err != nil { + if os.IsNotExist(err) { + file, err := os.Create(dbFilePath) + if err != nil { + return nil, err + } + + err = file.Close() + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + return gorm.Open(sqlite.Open(dbFilePath), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) +} diff --git a/go.mod b/go.mod index 2ffafb9e..d53aa885 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/gravitl/netmaker -go 1.23 +go 1.23.0 + +toolchain go1.23.7 require ( github.com/blang/semver v3.5.1+incompatible @@ -18,11 +20,11 @@ require ( github.com/stretchr/testify v1.10.0 github.com/txn2/txeh v1.5.5 go.uber.org/automaxprocs v1.6.0 - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.36.0 golang.org/x/net v0.34.0 // indirect golang.org/x/oauth2 v0.24.0 - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb gopkg.in/yaml.v3 v3.0.1 ) @@ -53,11 +55,20 @@ require ( github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/kr/text v0.2.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/seancfoley/bintree v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gorm.io/driver/postgres v1.5.11 // indirect + gorm.io/driver/sqlite v1.5.7 // indirect + gorm.io/gorm v1.25.12 // indirect ) require ( @@ -69,5 +80,5 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sync v0.10.0 // indirect + golang.org/x/sync v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index da0ae967..c99f9eb5 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,21 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -90,6 +103,7 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -103,6 +117,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -121,6 +137,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -131,6 +149,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -144,6 +164,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -161,3 +183,9 @@ gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314= +gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= +gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= diff --git a/migrate/migrate_schema.go b/migrate/migrate_schema.go new file mode 100644 index 00000000..330c883f --- /dev/null +++ b/migrate/migrate_schema.go @@ -0,0 +1,183 @@ +package migrate + +import ( + "context" + "errors" + "fmt" + "github.com/gravitl/netmaker/database" + "github.com/gravitl/netmaker/db" + "github.com/gravitl/netmaker/schema" + "github.com/gravitl/netmaker/servercfg" + "gorm.io/gorm" + "os" + "path/filepath" +) + +// ToSQLSchema migrates the data from key-value +// db to sql db. +// +// This function archives the old data and does not +// delete it. +// +// Based on the db server, the archival is done in the +// following way: +// +// 1. Sqlite: Moves the old data to a +// netmaker_archive.db file. +// +// 2. Postgres: Moves the data to a netmaker_archive +// schema within the same database. +func ToSQLSchema() error { + // initialize sql schema db. + err := db.InitializeDB(schema.ListModels()...) + if err != nil { + return err + } + + // migrate, if not done already. + err = migrate() + if err != nil { + return err + } + + // archive key-value schema db, if not done already. + // ignore errors. + _ = archive() + + return nil +} + +func migrate() error { + // begin a new transaction. + dbctx := db.BeginTx(context.TODO()) + commit := false + defer func() { + if commit { + db.FromContext(dbctx).Commit() + } else { + db.FromContext(dbctx).Rollback() + } + }() + + // check if migrated already. + migrationJob := &schema.Job{ + ID: "migration-v1.0.0", + } + err := migrationJob.Get(dbctx) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + // initialize key-value schema db. + err := database.InitializeDatabase() + if err != nil { + return err + } + defer database.CloseDB() + + // migrate. + // TODO: add migration code. + + // mark migration job completed. + err = migrationJob.Create(dbctx) + if err != nil { + return err + } + + commit = true + } + + return nil +} + +func archive() error { + dbServer := servercfg.GetDB() + if dbServer != "sqlite" && dbServer != "postgres" { + return nil + } + + // begin a new transaction. + dbctx := db.BeginTx(context.TODO()) + commit := false + defer func() { + if commit { + db.FromContext(dbctx).Commit() + } else { + db.FromContext(dbctx).Rollback() + } + }() + + // check if key-value schema db archived already. + archivalJob := &schema.Job{ + ID: "archival-v1.0.0", + } + err := archivalJob.Get(dbctx) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + // archive. + switch dbServer { + case "sqlite": + err = sqliteArchiveOldData() + default: + err = pgArchiveOldData() + } + if err != nil { + return err + } + + // mark archival job completed. + err = archivalJob.Create(dbctx) + if err != nil { + return err + } + + commit = true + } else { + // remove the residual + if dbServer == "sqlite" { + _ = os.Remove(filepath.Join("data", "netmaker.db")) + } + } + + return nil +} + +func sqliteArchiveOldData() error { + oldDBFilePath := filepath.Join("data", "netmaker.db") + archiveDBFilePath := filepath.Join("data", "netmaker_archive.db") + + // check if netmaker_archive.db exist. + _, err := os.Stat(archiveDBFilePath) + if err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + + // rename old db file to netmaker_archive.db. + return os.Rename(oldDBFilePath, archiveDBFilePath) +} + +func pgArchiveOldData() error { + _, err := database.PGDB.Exec("CREATE SCHEMA IF NOT EXISTS netmaker_archive") + if err != nil { + return err + } + + for _, table := range database.Tables { + _, err := database.PGDB.Exec( + fmt.Sprintf( + "ALTER TABLE public.%s SET SCHEMA netmaker_archive", + table, + ), + ) + if err != nil { + return err + } + } + return nil +} diff --git a/schema/jobs.go b/schema/jobs.go new file mode 100644 index 00000000..1e9e13f6 --- /dev/null +++ b/schema/jobs.go @@ -0,0 +1,36 @@ +package schema + +import ( + "context" + "github.com/gravitl/netmaker/db" + "time" +) + +// Job represents a task that netmaker server +// wants to do. +// +// Ideally, a jobs table should have details +// about its type, status, who initiated it, +// etc. But, for now, the table only contains +// records of jobs that have been done, so +// that it is easier to prevent a task from +// being executed again. +type Job struct { + ID string `gorm:"id;primary_key"` + CreatedAt time.Time `gorm:"created_at"` +} + +// TableName returns the name of the jobs table. +func (j *Job) TableName() string { + return "jobs" +} + +// Create creates a job record in the jobs table. +func (j *Job) Create(ctx context.Context) error { + return db.FromContext(ctx).Table(j.TableName()).Create(j).Error +} + +// Get returns a job record with the given Job.ID. +func (j *Job) Get(ctx context.Context) error { + return db.FromContext(ctx).Table(j.TableName()).Where("id = ?", j.ID).First(j).Error +} diff --git a/schema/models.go b/schema/models.go new file mode 100644 index 00000000..b03572f7 --- /dev/null +++ b/schema/models.go @@ -0,0 +1,8 @@ +package schema + +// ListModels lists all the models in this schema. +func ListModels() []interface{} { + return []interface{}{ + &Job{}, + } +}