From 7131c1c95c9624f47d6ed126648c67290d052df2 Mon Sep 17 00:00:00 2001 From: divyam234 Date: Fri, 18 Aug 2023 01:13:36 +0530 Subject: [PATCH] migrated to pressly as atlas go was causing errors to some users --- README.md | 23 +- database/database.go | 15 +- database/migrations/20230817172319_init.sql | 66 ++++ database/migrations/20230817172325_tables.sql | 88 +++++ .../migrations/20230817172329_functions.sql | 188 +++++++++++ database/migrations/atlas.sum | 2 - database/migrations/schema.sql | 314 ------------------ docker-compose.yml | 10 +- go.mod | 3 +- go.sum | 6 +- 10 files changed, 379 insertions(+), 336 deletions(-) create mode 100644 database/migrations/20230817172319_init.sql create mode 100644 database/migrations/20230817172325_tables.sql create mode 100644 database/migrations/20230817172329_functions.sql delete mode 100644 database/migrations/atlas.sum delete mode 100644 database/migrations/schema.sql diff --git a/README.md b/README.md index 22444f7..73332cb 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,6 @@ cd teldrive - Create the `.env` file with your variables and start your container. - **If you are deploying without https replace nginx.conf with nginx_nossl.conf in docker-compose.yml**. -- **Also Replace #DBURL with POSTGRES URL to RUN first time migrations and add ?search_path=public to postgres url at end so that migrations don't error out**. **It should look like this below if you are not using https.** ```yml @@ -59,17 +58,20 @@ git clone https://github.com/divyam234/teldrive - Fork UI Repo given above and Deploy it to Vercel. - Download release binary of teldrive from releases section. -- .env file will be same as mentioned above.. -- You have to run migrations manually for first run for that you have to download altasgo from [here](https://atlasgo.io/getting-started). - +- .env file will be same as mentioned above and additionally set variables mentioned below.As vercel app is hosted on https so we need local server also on https so that cookies works. ```shell -atlas migrate apply \ - --url postgres://root:pass@:5432/db1?search_path=public \ - --dir file://database/migrations --allow-dirty +HTTPS=true +COOKIE_SAME_SITE=false ``` +- Generate https cert and key for localhost using mkcert and put these in sslcerts directory where executable is present. + +- If you are using windows make sure to add cert as trusted using mkcert or manually.(You can see mkcert cli how to add that) + +- Rename generated cert and key as cert.pem and key.pem respectively. + - Now run the teldrive executable from releases. -- Finally change API URL from UI deployed on vercel to http://localhost:8080 in settings. +- Finally change API URL from UI deployed on vercel to https://localhost:8080 in settings. ## Setting up things @@ -80,6 +82,8 @@ An example of `.env` file: API_ID=1234 API_HASH=abc CHANNEL_ID=1234 +HTTPS=false +COOKIE_SAME_SITE=true JWT_SECRET=abc DATABASE_URL=abc MULTI_CLIENT=true # true or false here @@ -104,6 +108,9 @@ Before running the bot, you will need to set up the following mandatory variable ### Optional Vars In addition to the mandatory variables, you can also set the following optional variables: +- `HTTPS` : Only needed when frontend is deployed on vercel. + +- `COOKIE_SAME_SITE` : Only needed when frontend is deployed on vercel. - `MULTI_CLIENT` : Enable or Disable Multi Token mode. If true you have pass atleast one Multi Token - `MULTI_TOKEN[1....]` : Recommended to add atleast 10-12 tokens ### For making use of Multi-Client support diff --git a/database/database.go b/database/database.go index 3644f22..ae3b4d0 100644 --- a/database/database.go +++ b/database/database.go @@ -6,6 +6,7 @@ import ( "time" "github.com/divyam234/teldrive/utils" + "github.com/pressly/goose" "gorm.io/driver/postgres" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -52,9 +53,21 @@ func InitDB() { sqlDB.SetMaxOpenConns(100) sqlDB.SetConnMaxLifetime(time.Hour) - + migrate() go func() { DB.Exec(`create collation if not exists numeric (provider = icu, locale = 'en@colnumeric=yes');`) + migrate() }() } + +func migrate() { + + if err := goose.SetDialect("postgres"); err != nil { + panic(err) + } + db, _ := DB.DB() + if err := goose.Up(db, "database/migrations"); err != nil { + panic(err) + } +} diff --git a/database/migrations/20230817172319_init.sql b/database/migrations/20230817172319_init.sql new file mode 100644 index 0000000..69fc7ad --- /dev/null +++ b/database/migrations/20230817172319_init.sql @@ -0,0 +1,66 @@ +-- +goose Up +create extension if not exists pgcrypto; + +create extension if not exists btree_gin; + +create schema if not exists teldrive; + +create collation if not exists numeric (provider = icu, locale = 'en@colnumeric=yes'); + +-- +goose StatementBegin +create or replace +function teldrive.generate_uid(size int) returns text language plpgsql as $$ +declare characters text := 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + +bytes bytea := gen_random_bytes(size); + +l int := length(characters); + +i int := 0; + +output text := ''; + +begin while i < size loop output := output || substr(characters, +get_byte(bytes, +i) % l + 1, +1); + +i := i + 1; +end loop; + +return output; +end; + +$$; +-- +goose StatementEnd + +-- +goose StatementBegin +create or replace +function teldrive.get_tsvector(t text) returns tsvector language plpgsql immutable as $$ +declare res tsvector := to_tsvector(regexp_replace(t, +'[^A-Za-z0-9 ]', +' ', +'g')); + +begin return res; +end; + +$$; +-- +goose StatementEnd + +-- +goose StatementBegin +create or replace +function teldrive.get_tsquery(t text) returns tsquery language plpgsql immutable as $$ +declare res tsquery = concat( +plainto_tsquery(regexp_replace(t, +'[^A-Za-z0-9 ]', +' ', +'g')), +':*' +)::tsquery; + +begin return res; +end; + +$$; +-- +goose StatementEnd diff --git a/database/migrations/20230817172325_tables.sql b/database/migrations/20230817172325_tables.sql new file mode 100644 index 0000000..d38a323 --- /dev/null +++ b/database/migrations/20230817172325_tables.sql @@ -0,0 +1,88 @@ +-- +goose Up +create table teldrive.files ( +id text primary key not null default teldrive.generate_uid(16), +name text not null, +type text not null, +mime_type text not null, +path text null, +size bigint null, +starred bool not null, +depth integer null, +user_id bigint not null, +parent_id text null, +status text default 'active'::text, +channel_id bigint null, +parts jsonb null, +created_at timestamp not null default timezone('utc'::text, +now()), +updated_at timestamp not null default timezone('utc'::text, +now()), +constraint unique_file unique (name, +parent_id,user_id) +); + +create table teldrive.uploads ( + id text not null primary key default teldrive.generate_uid(16), + upload_id text not null, + name text not null, + part_no int4 not null, + part_id int4 not null, + total_parts int4 not null, + channel_id int8 not null, + size int8 not null, + created_at timestamp null default timezone('utc'::text, +now()) +); + +create table teldrive.users ( + user_id int4 not null primary key, + name text null, + user_name text null, + is_premium bool not null, + tg_session text not null, + settings jsonb null, + created_at timestamptz not null default timezone('utc'::text, +now()), + updated_at timestamptz not null default timezone('utc'::text, +now()) +); + +create collation if not exists numeric (provider = icu, locale = 'en@colnumeric=yes'); +create index name_search_idx on +teldrive.files + using gin (teldrive.get_tsvector(name), +updated_at); + +create index name_numeric_idx on +teldrive.files(name collate numeric nulls first); + +create index parent_name_numeric_idx on +teldrive.files (parent_id, +name collate numeric desc); + +create index path_idx on +teldrive.files (path); + +create index parent_idx on +teldrive.files (parent_id); + +create index starred_updated_at_idx on +teldrive.files (starred, +updated_at desc); + +create index status_idx on teldrive.files (status); + +create index user_id_idx on teldrive.files (user_id); + +-- +goose Down +drop table if exists teldrive.files; +drop table if exists teldrive.uploads; +drop table if exists teldrive.users; +drop index if exists teldrive.name_search_idx; +drop index if exists teldrive.name_numeric_idx; +drop index if exists teldrive.parent_name_numeric_idx; +drop index if exists teldrive.path_idx; +drop index if exists teldrive.parent_idx; +drop index if exists teldrive.starred_updated_at_idx; +drop index if exists teldrive.status_idx; +drop index if exists teldrive.user_id_idx; \ No newline at end of file diff --git a/database/migrations/20230817172329_functions.sql b/database/migrations/20230817172329_functions.sql new file mode 100644 index 0000000..ff7d3c0 --- /dev/null +++ b/database/migrations/20230817172329_functions.sql @@ -0,0 +1,188 @@ +-- +goose Up + +-- +goose StatementBegin +create procedure teldrive.update_size() language plpgsql as $$ +declare rec record; + +total_size bigint; + +begin + +for rec in +select + id +from + files +where + type = 'folder' +order by + depth desc loop total_size := ( + select + sum(size) as total_size + from + teldrive.files + where + parent_id = rec.id +); + +update + teldrive.files +set + size = total_size +where + id = rec.id; +end loop; +end; + +$$; +-- +goose StatementEnd + + +-- +goose StatementBegin +create function teldrive.update_folder(folder_id text, +new_name text default null, +new_path text default null) + returns setof teldrive.files + language plpgsql +as $$ +declare folder record; + +path_items text []; + +begin + +if new_path is null then +select + * +into + folder +from + teldrive.files +where + id = folder_id; + +path_items := string_to_array(folder.path, +'/'); + +path_items [array_length(path_items, +1)] := new_name; + +new_path := array_to_string(path_items, +'/'); +end if; + +update + teldrive.files +set + path = new_path, + name = new_name +where + id = folder_id; + +for folder in +select + * +from + teldrive.files +where + type = 'folder' + and parent_id = folder_id loop call teldrive.update_folder( + folder.id, + folder.name, + concat(new_path, + '/', + folder.name) +); +end loop; + +return query +select + * +from + teldrive.files +where + id = folder_id; +end; + +$$ +; +-- +goose StatementEnd + +-- +goose StatementBegin +create procedure teldrive.delete_files(in file_ids text[], +in op text default 'bulk') + language plpgsql +as $$ + declare + rec record; + +begin + if op = 'bulk' then + for rec in +select + id, + type +from + teldrive.files +where + id = any (file_ids) + loop + if rec.type = 'folder' then + call teldrive.delete_files(array [rec.id], + 'single'); + +delete +from + teldrive.files +where + id = rec.id; +else + update + teldrive.files +set + status = 'pending_deletion' +where + id = rec.id; +end if; +end loop; +else + + for rec in +select + id, + type +from + teldrive.files +where + parent_id = file_ids[1] + loop + if rec.type = 'folder' then + call teldrive.delete_files(array [rec.id], + 'single'); + +delete +from + teldrive.files +where + id = rec.id; +else + update + teldrive.files +set + status = 'pending_deletion' +where + id = rec.id; +end if; +end loop; +end if; +end; + +$$ +; +-- +goose StatementEnd + + +-- +goose Down +drop procedure if exists teldrive.update_size; +drop function if exists teldrive.update_folder; +drop procedure if exists teldrive.delete_files; \ No newline at end of file diff --git a/database/migrations/atlas.sum b/database/migrations/atlas.sum deleted file mode 100644 index 38fc71e..0000000 --- a/database/migrations/atlas.sum +++ /dev/null @@ -1,2 +0,0 @@ -h1:zrLeo6SFfYzfnY454gOuslG0TAveXl4Fg0Z1DmXMADI= -schema.sql h1:UdvfNm4Fb8bC67XsF6HBuxhRZNTgUAf4fwpgrm2GY3k= diff --git a/database/migrations/schema.sql b/database/migrations/schema.sql deleted file mode 100644 index 9b8d16f..0000000 --- a/database/migrations/schema.sql +++ /dev/null @@ -1,314 +0,0 @@ -create extension if not exists pgcrypto; - -create extension if not exists btree_gin; - -create schema if not exists teldrive; - -set -search_path = 'teldrive'; - -create or replace -function generate_uid(size int) returns text as $$ -declare characters text := 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz0123456789'; - -bytes bytea := gen_random_bytes(size); - -l int := length(characters); - -i int := 0; - -output text := ''; - -begin while i < size loop output := output || substr(characters, -get_byte(bytes, -i) % l + 1, -1); - -i := i + 1; -end loop; - -return output; -end; - -$$ language plpgsql volatile; - -create or replace -function get_tsvector(t text) returns tsvector language plpgsql immutable as $$ -declare res tsvector := to_tsvector(regexp_replace(t, -'[^a-za-z0-9 ]', -' ', -'g')); - -begin return res; -end; - -$$; - -create or replace -function get_tsquery(t text) returns tsquery language plpgsql immutable as $$ -declare res tsquery = concat( -plainto_tsquery(regexp_replace(t, -'[^a-za-z0-9 ]', -' ', -'g')), -':*' -)::tsquery; - -begin return res; -end; - -$$; - -create table if not exists files ( -id text primary key not null default generate_uid(16), -name text not null, -type text not null, -mime_type text not null, -path text null, -size bigint null, -starred bool not null, -depth integer null, -user_id bigint not null, -parent_id text null, -status text DEFAULT 'active'::text, -channel_id bigint null, -parts jsonb NULL, -created_at timestamp not null default timezone('utc'::text, -now()), -updated_at timestamp not null default timezone('utc'::text, -now()), -constraint unique_file unique (name, -parent_id,user_id) -); - -create table uploads ( - id text not null primary key default generate_uid(16), - upload_id text not null, - name text not null, - part_no int4 not null, - part_id int4 not null, - total_parts int4 not null, - channel_id int8 not null, - size int8 not null, - created_at timestamp null default timezone('utc'::text, -now()) -); - -create table teldrive.users ( - user_id int4 not null primary key, - name text null, - user_name text null, - is_premium bool not null, - tg_session text not null, - settings jsonb null, - created_at timestamptz not null default timezone('utc'::text, -now()), - updated_at timestamptz not null default timezone('utc'::text, -now()) -); - - -create or replace -procedure update_size() language plpgsql as $$ -declare rec record; - -total_size bigint; - -begin - -for rec in -select - id -from - files -where - type = 'folder' -order by - depth desc loop total_size := ( - select - sum(size) as total_size - from - teldrive.files - where - parent_id = rec.id -); - -update - teldrive.files -set - size = total_size -where - id = rec.id; -end loop; -end; - -$$; - -create or replace -function teldrive.update_folder(folder_id text, -new_name text default null, -new_path text default null) - returns setof teldrive.files - language plpgsql -as $$ -declare folder record; - -path_items text []; - -begin - -if new_path is null then -select - * -into - folder -from - teldrive.files -where - id = folder_id; - -path_items := string_to_array(folder.path, -'/'); - -path_items [array_length(path_items, -1)] := new_name; - -new_path := array_to_string(path_items, -'/'); -end if; - -update - teldrive.files -set - path = new_path, - name = new_name -where - id = folder_id; - -for folder in -select - * -from - teldrive.files -where - type = 'folder' - and parent_id = folder_id loop call teldrive.update_folder( - folder.id, - folder.name, - concat(new_path, - '/', - folder.name) -); -end loop; - -return query -select - * -from - teldrive.files -where - id = folder_id; -end; - -$$ -; - -create or replace -procedure teldrive.delete_files(in file_ids text[], -in op text default 'bulk') - language plpgsql -as $$ - declare - rec record; - -begin - if op = 'bulk' then - for rec in -select - id, - type -from - teldrive.files -where - id = any (file_ids) - loop - if rec.type = 'folder' then - call teldrive.delete_files(array [rec.id], - 'single'); - -delete -from - teldrive.files -where - id = rec.id; -else - update - teldrive.files -set - status = 'pending_deletion' -where - id = rec.id; -end if; -end loop; -else - - for rec in -select - id, - type -from - teldrive.files -where - parent_id = file_ids[1] - loop - if rec.type = 'folder' then - call teldrive.delete_files(array [rec.id], - 'single'); - -delete -from - teldrive.files -where - id = rec.id; -else - update - teldrive.files -set - status = 'pending_deletion' -where - id = rec.id; -end if; -end loop; -end if; -end; - -$$ -; - -create collation if not exists numeric (provider = icu, locale = 'en@colnumeric=yes'); - -create index if not exists name_search_idx on -files - using gin (get_tsvector(name), -updated_at); - -create index if not exists name_numeric_idx on -files(name collate numeric nulls first); - -create index if not exists parent_name_numeric_idx on -files (parent_id, -name collate numeric desc); - -create index if not exists path_idx on -files (path); - -create index if not exists parent_idx on -files (parent_id); - -create index if not exists starred_updated_at_idx on -files (starred, -updated_at desc); - -create index if not exists status_idx on files (status); - -create index if not exists user_id_idx on files (user_id); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c12bdc3..78182fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: container_name: client ports: - 3000 - + nginx: image: nginx:alpine restart: always @@ -30,10 +30,4 @@ services: - 443:443 depends_on: - server - - client - migrate: - image: arigaio/atlas:latest - command: > - migrate apply --url #DBURL --allow-dirty - volumes: - - ./database/migrations/:/migrations \ No newline at end of file + - client \ No newline at end of file diff --git a/go.mod b/go.mod index 7280a2f..8ca3d0b 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/gotd/neo v0.1.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.3.1 // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -55,6 +55,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pressly/goose v2.7.0+incompatible github.com/segmentio/asm v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect diff --git a/go.sum b/go.sum index 784322b..f4331fd 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= -github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -129,6 +129,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ= +github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/quantumsheep/range-parser v1.1.0 h1:k4f1F58f8FF54FBYc9dYBRM+8JkAxFo11gC3IeMH4rU= github.com/quantumsheep/range-parser v1.1.0/go.mod h1:acv4Vt2PvpGvRsvGju7Gk2ahKluZJsIUNR69W53J22I= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=