2018-10-25 21:51:47 +08:00
|
|
|
DROP TYPE IF EXISTS list_type CASCADE; CREATE TYPE list_type AS ENUM ('public', 'private', 'temporary');
|
2019-12-01 20:18:36 +08:00
|
|
|
DROP TYPE IF EXISTS list_optin CASCADE; CREATE TYPE list_optin AS ENUM ('single', 'double');
|
2020-08-01 19:15:29 +08:00
|
|
|
DROP TYPE IF EXISTS subscriber_status CASCADE; CREATE TYPE subscriber_status AS ENUM ('enabled', 'disabled', 'blocklisted');
|
2018-10-25 21:51:47 +08:00
|
|
|
DROP TYPE IF EXISTS subscription_status CASCADE; CREATE TYPE subscription_status AS ENUM ('unconfirmed', 'confirmed', 'unsubscribed');
|
|
|
|
DROP TYPE IF EXISTS campaign_status CASCADE; CREATE TYPE campaign_status AS ENUM ('draft', 'running', 'scheduled', 'paused', 'cancelled', 'finished');
|
2020-02-03 15:48:26 +08:00
|
|
|
DROP TYPE IF EXISTS campaign_type CASCADE; CREATE TYPE campaign_type AS ENUM ('regular', 'optin');
|
2021-04-11 18:43:43 +08:00
|
|
|
DROP TYPE IF EXISTS content_type CASCADE; CREATE TYPE content_type AS ENUM ('richtext', 'html', 'plain', 'markdown');
|
2021-05-25 01:11:48 +08:00
|
|
|
DROP TYPE IF EXISTS bounce_type CASCADE; CREATE TYPE bounce_type AS ENUM ('soft', 'hard', 'complaint');
|
2022-07-02 18:00:17 +08:00
|
|
|
DROP TYPE IF EXISTS template_type CASCADE; CREATE TYPE template_type AS ENUM ('campaign', 'tx');
|
2024-06-16 16:20:04 +08:00
|
|
|
DROP TYPE IF EXISTS user_type CASCADE; CREATE TYPE user_type AS ENUM ('user', 'api');
|
2024-05-07 13:38:31 +08:00
|
|
|
DROP TYPE IF EXISTS user_status CASCADE; CREATE TYPE user_status AS ENUM ('enabled', 'disabled');
|
2024-09-02 20:13:56 +08:00
|
|
|
DROP TYPE IF EXISTS role_type CASCADE; CREATE TYPE role_type AS ENUM ('user', 'list');
|
2018-10-25 21:51:47 +08:00
|
|
|
|
2024-05-07 13:38:31 +08:00
|
|
|
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
2024-04-10 02:50:48 +08:00
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
-- subscribers
|
|
|
|
DROP TABLE IF EXISTS subscribers CASCADE;
|
|
|
|
CREATE TABLE subscribers (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
uuid uuid NOT NULL UNIQUE,
|
|
|
|
email TEXT NOT NULL UNIQUE,
|
|
|
|
name TEXT NOT NULL,
|
2018-12-26 18:01:30 +08:00
|
|
|
attribs JSONB NOT NULL DEFAULT '{}',
|
2018-11-05 19:29:09 +08:00
|
|
|
status subscriber_status NOT NULL DEFAULT 'enabled',
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2018-12-26 18:01:30 +08:00
|
|
|
DROP INDEX IF EXISTS idx_subs_email; CREATE UNIQUE INDEX idx_subs_email ON subscribers(LOWER(email));
|
|
|
|
DROP INDEX IF EXISTS idx_subs_status; CREATE INDEX idx_subs_status ON subscribers(status);
|
Fix and refactor subscriber batch fetching in campaign processing.
This has been a hair-pulling rabbit hole of an issue. #1931 and others.
When the `next-campaign-subscribers` query that fetches $n subscribers
per batch for a campaign returns no results, the manager assumes
that the campaign is done and marks as finished.
Marathon debugging revealed fundamental flaws in qyery's logic that
would incorrectly return 0 rows under certain conditions.
- Based on the "layout" of subscribers for eg: a series of blocklisted
subscribers between confirmed subscribers.
A series of unconfirmed subscribers in a batch belonging to a double
opt-in list.
- Bulk import blocklisting users, but not marking their subscriptions
as 'unsubscribed'.
- Conditions spread across multiple CTEs resulted in returning an
arbitrary number of rows and $N per batch as the selected $N rows
would get filtered out elsewhere, possibly even becoming 0.
After fixing this and testing it on our prod instance that has
15 million subscribers and ~70 million subscriptions in the
`subscriber_lists` table, ended up discovered significant inefficiences
in Postgres query planning. When `subscriber_lists` and campaign list IDs
are joined dynamically (CTE or ANY() or any kind of JOIN that involves)
a query, the Postgres query planner is unable to use the right indexes.
After testing dozens of approaches, discovered that statically passing
the values to join on (hardcoding or passing via parametrized $1 vars),
the query uses the right indexes. The difference is staggering.
For the particular scenario on our large prod DB to pull a batch,
~15 seconds vs. ~50ms, a whopping 300x improvement!
This patch splits `next-campaign-subscribers` into two separate queries,
one which fetches campaign metadata and list_ids, whose values are then
passed statically to the next query to fetch subscribers by batch.
In addition, it fixes and refactors broken filtering and counting logic
in `create-campaign` and `next-campaign` queries.
Closes #1931, #1993, #1986.
2024-09-20 20:21:44 +08:00
|
|
|
DROP INDEX IF EXISTS idx_subs_id_status; CREATE INDEX idx_subs_id_status ON subscribers(id, status);
|
2024-01-12 00:53:39 +08:00
|
|
|
DROP INDEX IF EXISTS idx_subs_created_at; CREATE INDEX idx_subs_created_at ON subscribers(created_at);
|
|
|
|
DROP INDEX IF EXISTS idx_subs_updated_at; CREATE INDEX idx_subs_updated_at ON subscribers(updated_at);
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
-- lists
|
|
|
|
DROP TABLE IF EXISTS lists CASCADE;
|
|
|
|
CREATE TABLE lists (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
uuid uuid NOT NULL UNIQUE,
|
|
|
|
name TEXT NOT NULL,
|
|
|
|
type list_type NOT NULL,
|
2019-12-01 20:18:36 +08:00
|
|
|
optin list_optin NOT NULL DEFAULT 'single',
|
2018-10-25 21:51:47 +08:00
|
|
|
tags VARCHAR(100)[],
|
2022-11-01 23:29:21 +08:00
|
|
|
description TEXT NOT NULL DEFAULT '',
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2024-01-12 00:53:39 +08:00
|
|
|
DROP INDEX IF EXISTS idx_lists_type; CREATE INDEX idx_lists_type ON lists(type);
|
|
|
|
DROP INDEX IF EXISTS idx_lists_optin; CREATE INDEX idx_lists_optin ON lists(optin);
|
|
|
|
DROP INDEX IF EXISTS idx_lists_name; CREATE INDEX idx_lists_name ON lists(name);
|
|
|
|
DROP INDEX IF EXISTS idx_lists_created_at; CREATE INDEX idx_lists_created_at ON lists(created_at);
|
|
|
|
DROP INDEX IF EXISTS idx_lists_updated_at; CREATE INDEX idx_lists_updated_at ON lists(updated_at);
|
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
DROP TABLE IF EXISTS subscriber_lists CASCADE;
|
|
|
|
CREATE TABLE subscriber_lists (
|
|
|
|
subscriber_id INTEGER REFERENCES subscribers(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
|
|
list_id INTEGER NULL REFERENCES lists(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
2023-07-22 15:58:45 +08:00
|
|
|
meta JSONB NOT NULL DEFAULT '{}',
|
2018-10-25 21:51:47 +08:00
|
|
|
status subscription_status NOT NULL DEFAULT 'unconfirmed',
|
|
|
|
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
|
|
|
|
PRIMARY KEY(subscriber_id, list_id)
|
|
|
|
);
|
2018-12-26 18:01:30 +08:00
|
|
|
DROP INDEX IF EXISTS idx_sub_lists_sub_id; CREATE INDEX idx_sub_lists_sub_id ON subscriber_lists(subscriber_id);
|
|
|
|
DROP INDEX IF EXISTS idx_sub_lists_list_id; CREATE INDEX idx_sub_lists_list_id ON subscriber_lists(list_id);
|
|
|
|
DROP INDEX IF EXISTS idx_sub_lists_status; CREATE INDEX idx_sub_lists_status ON subscriber_lists(status);
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
-- templates
|
|
|
|
DROP TABLE IF EXISTS templates CASCADE;
|
|
|
|
CREATE TABLE templates (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
name TEXT NOT NULL,
|
2022-07-02 18:00:17 +08:00
|
|
|
type template_type NOT NULL DEFAULT 'campaign',
|
|
|
|
subject TEXT NOT NULL,
|
2018-10-25 21:51:47 +08:00
|
|
|
body TEXT NOT NULL,
|
|
|
|
is_default BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
|
|
|
CREATE UNIQUE INDEX ON templates (is_default) WHERE is_default = true;
|
|
|
|
|
|
|
|
|
|
|
|
-- campaigns
|
|
|
|
DROP TABLE IF EXISTS campaigns CASCADE;
|
|
|
|
CREATE TABLE campaigns (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
uuid uuid NOT NULL UNIQUE,
|
|
|
|
name TEXT NOT NULL,
|
|
|
|
subject TEXT NOT NULL,
|
|
|
|
from_email TEXT NOT NULL,
|
|
|
|
body TEXT NOT NULL,
|
2021-01-30 17:29:21 +08:00
|
|
|
altbody TEXT NULL,
|
2018-10-25 21:51:47 +08:00
|
|
|
content_type content_type NOT NULL DEFAULT 'richtext',
|
|
|
|
send_at TIMESTAMP WITH TIME ZONE,
|
2022-01-05 00:46:21 +08:00
|
|
|
headers JSONB NOT NULL DEFAULT '[]',
|
2018-10-25 21:51:47 +08:00
|
|
|
status campaign_status NOT NULL DEFAULT 'draft',
|
|
|
|
tags VARCHAR(100)[],
|
|
|
|
|
2020-02-03 15:48:26 +08:00
|
|
|
-- The subscription statuses of subscribers to which a campaign will be sent.
|
|
|
|
-- For opt-in campaigns, this will be 'unsubscribed'.
|
|
|
|
type campaign_type DEFAULT 'regular',
|
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
-- The ID of the messenger backend used to send this campaign.
|
|
|
|
messenger TEXT NOT NULL,
|
|
|
|
template_id INTEGER REFERENCES templates(id) ON DELETE SET DEFAULT DEFAULT 1,
|
|
|
|
|
|
|
|
-- Progress and stats.
|
|
|
|
to_send INT NOT NULL DEFAULT 0,
|
|
|
|
sent INT NOT NULL DEFAULT 0,
|
|
|
|
max_subscriber_id INT NOT NULL DEFAULT 0,
|
|
|
|
last_subscriber_id INT NOT NULL DEFAULT 0,
|
|
|
|
|
2022-11-03 13:37:26 +08:00
|
|
|
-- Publishing.
|
|
|
|
archive BOOLEAN NOT NULL DEFAULT false,
|
2024-01-10 02:04:08 +08:00
|
|
|
archive_slug TEXT NULL UNIQUE,
|
2022-11-03 13:37:26 +08:00
|
|
|
archive_template_id INTEGER REFERENCES templates(id) ON DELETE SET DEFAULT DEFAULT 1,
|
|
|
|
archive_meta JSONB NOT NULL DEFAULT '{}',
|
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
started_at TIMESTAMP WITH TIME ZONE,
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2024-01-12 00:53:39 +08:00
|
|
|
DROP INDEX IF EXISTS idx_camps_status; CREATE INDEX idx_camps_status ON campaigns(status);
|
|
|
|
DROP INDEX IF EXISTS idx_camps_name; CREATE INDEX idx_camps_name ON campaigns(name);
|
|
|
|
DROP INDEX IF EXISTS idx_camps_created_at; CREATE INDEX idx_camps_created_at ON campaigns(created_at);
|
|
|
|
DROP INDEX IF EXISTS idx_camps_updated_at; CREATE INDEX idx_camps_updated_at ON campaigns(updated_at);
|
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
DROP TABLE IF EXISTS campaign_lists CASCADE;
|
|
|
|
CREATE TABLE campaign_lists (
|
2021-09-17 22:48:53 +08:00
|
|
|
id BIGSERIAL PRIMARY KEY,
|
2018-10-25 21:51:47 +08:00
|
|
|
campaign_id INTEGER NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
|
|
|
|
|
|
-- Lists may be deleted, so list_id is nullable
|
|
|
|
-- and a copy of the original list name is maintained here.
|
|
|
|
list_id INTEGER NULL REFERENCES lists(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
|
|
|
list_name TEXT NOT NULL DEFAULT ''
|
|
|
|
);
|
|
|
|
CREATE UNIQUE INDEX ON campaign_lists (campaign_id, list_id);
|
2018-12-26 18:01:30 +08:00
|
|
|
DROP INDEX IF EXISTS idx_camp_lists_camp_id; CREATE INDEX idx_camp_lists_camp_id ON campaign_lists(campaign_id);
|
|
|
|
DROP INDEX IF EXISTS idx_camp_lists_list_id; CREATE INDEX idx_camp_lists_list_id ON campaign_lists(list_id);
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
DROP TABLE IF EXISTS campaign_views CASCADE;
|
|
|
|
CREATE TABLE campaign_views (
|
2021-09-17 22:25:38 +08:00
|
|
|
id BIGSERIAL PRIMARY KEY,
|
2018-11-02 15:50:32 +08:00
|
|
|
campaign_id INTEGER NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
2018-10-25 21:51:47 +08:00
|
|
|
|
2018-11-02 15:50:32 +08:00
|
|
|
-- Subscribers may be deleted, but the view counts should remain.
|
2018-10-25 21:51:47 +08:00
|
|
|
subscriber_id INTEGER NULL REFERENCES subscribers(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2018-12-26 18:01:30 +08:00
|
|
|
DROP INDEX IF EXISTS idx_views_camp_id; CREATE INDEX idx_views_camp_id ON campaign_views(campaign_id);
|
|
|
|
DROP INDEX IF EXISTS idx_views_subscriber_id; CREATE INDEX idx_views_subscriber_id ON campaign_views(subscriber_id);
|
2021-09-18 18:16:22 +08:00
|
|
|
DROP INDEX IF EXISTS idx_views_date; CREATE INDEX idx_views_date ON campaign_views((TIMEZONE('UTC', created_at)::DATE));
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
-- media
|
|
|
|
DROP TABLE IF EXISTS media CASCADE;
|
|
|
|
CREATE TABLE media (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
uuid uuid NOT NULL UNIQUE,
|
2020-08-03 21:32:23 +08:00
|
|
|
provider TEXT NOT NULL DEFAULT '',
|
2018-10-25 21:51:47 +08:00
|
|
|
filename TEXT NOT NULL,
|
2023-05-18 19:25:59 +08:00
|
|
|
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
|
2018-10-25 21:51:47 +08:00
|
|
|
thumb TEXT NOT NULL,
|
2022-10-03 01:34:51 +08:00
|
|
|
meta JSONB NOT NULL DEFAULT '{}',
|
2018-10-25 21:51:47 +08:00
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
|
|
|
|
2023-05-18 19:25:59 +08:00
|
|
|
-- campaign_media
|
|
|
|
DROP TABLE IF EXISTS campaign_media CASCADE;
|
|
|
|
CREATE TABLE campaign_media (
|
|
|
|
campaign_id INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
|
|
|
|
|
|
-- Media items may be deleted, so media_id is nullable
|
|
|
|
-- and a copy of the original name is maintained here.
|
|
|
|
media_id INTEGER NULL REFERENCES media(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
|
|
|
|
|
|
|
filename TEXT NOT NULL DEFAULT ''
|
|
|
|
);
|
|
|
|
DROP INDEX IF EXISTS idx_camp_media_id; CREATE UNIQUE INDEX idx_camp_media_id ON campaign_media (campaign_id, media_id);
|
|
|
|
DROP INDEX IF EXISTS idx_camp_media_camp_id; CREATE INDEX idx_camp_media_camp_id ON campaign_media(campaign_id);
|
|
|
|
|
2023-05-21 17:49:12 +08:00
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
-- links
|
|
|
|
DROP TABLE IF EXISTS links CASCADE;
|
|
|
|
CREATE TABLE links (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
uuid uuid NOT NULL UNIQUE,
|
2018-10-31 20:54:21 +08:00
|
|
|
url TEXT NOT NULL UNIQUE,
|
2018-10-25 21:51:47 +08:00
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
|
|
|
|
|
|
|
DROP TABLE IF EXISTS link_clicks CASCADE;
|
|
|
|
CREATE TABLE link_clicks (
|
2021-09-17 22:25:38 +08:00
|
|
|
id BIGSERIAL PRIMARY KEY,
|
2020-10-24 16:33:37 +08:00
|
|
|
campaign_id INTEGER NULL REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
|
|
link_id INTEGER NOT NULL REFERENCES links(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
-- Subscribers may be deleted, but the link counts should remain.
|
|
|
|
subscriber_id INTEGER NULL REFERENCES subscribers(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2018-12-26 18:01:30 +08:00
|
|
|
DROP INDEX IF EXISTS idx_clicks_camp_id; CREATE INDEX idx_clicks_camp_id ON link_clicks(campaign_id);
|
|
|
|
DROP INDEX IF EXISTS idx_clicks_link_id; CREATE INDEX idx_clicks_link_id ON link_clicks(link_id);
|
|
|
|
DROP INDEX IF EXISTS idx_clicks_sub_id; CREATE INDEX idx_clicks_sub_id ON link_clicks(subscriber_id);
|
2021-09-18 18:16:22 +08:00
|
|
|
DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX idx_clicks_date ON link_clicks((TIMEZONE('UTC', created_at)::DATE));
|
2020-07-08 19:00:14 +08:00
|
|
|
|
|
|
|
-- settings
|
|
|
|
DROP TABLE IF EXISTS settings CASCADE;
|
|
|
|
CREATE TABLE settings (
|
|
|
|
key TEXT NOT NULL UNIQUE,
|
|
|
|
value JSONB NOT NULL DEFAULT '{}',
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
|
|
|
DROP INDEX IF EXISTS idx_settings_key; CREATE INDEX idx_settings_key ON settings(key);
|
|
|
|
INSERT INTO settings (key, value) VALUES
|
2022-11-10 23:49:53 +08:00
|
|
|
('app.site_name', '"Mailing list"'),
|
2020-08-01 20:24:51 +08:00
|
|
|
('app.root_url', '"http://localhost:9000"'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('app.favicon_url', '""'),
|
|
|
|
('app.from_email', '"listmonk <noreply@listmonk.yoursite.com>"'),
|
2022-02-16 16:09:38 +08:00
|
|
|
('app.logo_url', '""'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('app.concurrency', '10'),
|
|
|
|
('app.message_rate', '10'),
|
|
|
|
('app.batch_size', '1000'),
|
|
|
|
('app.max_send_errors', '1000'),
|
2021-01-24 14:49:26 +08:00
|
|
|
('app.message_sliding_window', 'false'),
|
|
|
|
('app.message_sliding_window_duration', '"1h"'),
|
|
|
|
('app.message_sliding_window_rate', '10000'),
|
2024-01-12 00:53:39 +08:00
|
|
|
('app.cache_slow_queries', 'false'),
|
|
|
|
('app.cache_slow_queries_interval', '"0 3 * * *"'),
|
2022-11-11 00:54:15 +08:00
|
|
|
('app.enable_public_archive', 'true'),
|
2021-01-31 18:49:39 +08:00
|
|
|
('app.enable_public_subscription_page', 'true'),
|
2023-04-08 12:09:10 +08:00
|
|
|
('app.enable_public_archive_rss_content', 'true'),
|
2021-09-25 13:08:13 +08:00
|
|
|
('app.send_optin_confirmation', 'true'),
|
2021-05-16 19:24:55 +08:00
|
|
|
('app.check_updates', 'true'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('app.notify_emails', '["admin1@mysite.com", "admin2@mysite.com"]'),
|
2020-12-19 18:55:52 +08:00
|
|
|
('app.lang', '"en"'),
|
2020-10-18 20:03:34 +08:00
|
|
|
('privacy.individual_tracking', 'false'),
|
2020-08-01 20:24:51 +08:00
|
|
|
('privacy.unsubscribe_header', 'true'),
|
2020-08-01 19:15:29 +08:00
|
|
|
('privacy.allow_blocklist', 'true'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('privacy.allow_export', 'true'),
|
|
|
|
('privacy.allow_wipe', 'true'),
|
2022-10-19 00:14:57 +08:00
|
|
|
('privacy.allow_preferences', 'true'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('privacy.exportable', '["profile", "subscriptions", "campaign_views", "link_clicks"]'),
|
2021-09-25 15:27:55 +08:00
|
|
|
('privacy.domain_blocklist', '[]'),
|
2023-07-22 15:58:45 +08:00
|
|
|
('privacy.record_optin_ip', 'false'),
|
2023-01-24 00:20:10 +08:00
|
|
|
('security.enable_captcha', 'false'),
|
|
|
|
('security.captcha_key', '""'),
|
|
|
|
('security.captcha_secret', '""'),
|
2024-04-02 12:58:07 +08:00
|
|
|
('security.oidc', '{"enabled": false, "provider_url": "", "client_id": "", "client_secret": ""}'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('upload.provider', '"filesystem"'),
|
2023-05-18 19:25:59 +08:00
|
|
|
('upload.max_file_size', '5000'),
|
|
|
|
('upload.extensions', '["jpg","jpeg","png","gif","svg","*"]'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('upload.filesystem.upload_path', '"uploads"'),
|
|
|
|
('upload.filesystem.upload_uri', '"/uploads"'),
|
2021-08-15 14:45:00 +08:00
|
|
|
('upload.s3.url', '"https://ap-south-1.s3.amazonaws.com"'),
|
2022-01-15 23:50:32 +08:00
|
|
|
('upload.s3.public_url', '""'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('upload.s3.aws_access_key_id', '""'),
|
|
|
|
('upload.s3.aws_secret_access_key', '""'),
|
2021-08-15 14:45:00 +08:00
|
|
|
('upload.s3.aws_default_region', '"ap-south-1"'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('upload.s3.bucket', '""'),
|
|
|
|
('upload.s3.bucket_domain', '""'),
|
|
|
|
('upload.s3.bucket_path', '"/"'),
|
|
|
|
('upload.s3.bucket_type', '"public"'),
|
2023-09-19 17:15:51 +08:00
|
|
|
('upload.s3.expiry', '"167h"'),
|
2020-07-08 19:00:14 +08:00
|
|
|
('smtp',
|
2022-01-03 21:58:36 +08:00
|
|
|
'[{"enabled":true, "host":"smtp.yoursite.com","port":25,"auth_protocol":"cram","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_type":"STARTTLS","tls_skip_verify":false,"email_headers":[]},
|
|
|
|
{"enabled":false, "host":"smtp.gmail.com","port":465,"auth_protocol":"login","username":"username@gmail.com","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_type":"TLS","tls_skip_verify":false,"email_headers":[]}]'),
|
2021-05-25 01:11:48 +08:00
|
|
|
('messengers', '[]'),
|
|
|
|
('bounce.enabled', 'false'),
|
|
|
|
('bounce.webhooks_enabled', 'false'),
|
2024-01-24 12:49:13 +08:00
|
|
|
('bounce.actions', '{"soft": {"count": 2, "action": "none"}, "hard": {"count": 1, "action": "blocklist"}, "complaint" : {"count": 1, "action": "blocklist"}}'),
|
2021-05-25 01:11:48 +08:00
|
|
|
('bounce.ses_enabled', 'false'),
|
|
|
|
('bounce.sendgrid_enabled', 'false'),
|
|
|
|
('bounce.sendgrid_key', '""'),
|
2023-08-31 23:57:34 +08:00
|
|
|
('bounce.postmark', '{"enabled": false, "username": "", "password": ""}'),
|
2021-05-25 01:11:48 +08:00
|
|
|
('bounce.mailboxes',
|
2021-12-18 18:08:42 +08:00
|
|
|
'[{"enabled":false, "type": "pop", "host":"pop.yoursite.com","port":995,"auth_protocol":"userpass","username":"username","password":"password","return_path": "bounce@listmonk.yoursite.com","scan_interval":"15m","tls_enabled":true,"tls_skip_verify":false}]'),
|
|
|
|
('appearance.admin.custom_css', '""'),
|
|
|
|
('appearance.admin.custom_js', '""'),
|
|
|
|
('appearance.public.custom_css', '""'),
|
|
|
|
('appearance.public.custom_js', '""');
|
2021-05-25 01:11:48 +08:00
|
|
|
|
|
|
|
-- bounces
|
|
|
|
DROP TABLE IF EXISTS bounces CASCADE;
|
|
|
|
CREATE TABLE bounces (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
subscriber_id INTEGER NOT NULL REFERENCES subscribers(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
|
|
campaign_id INTEGER NULL REFERENCES campaigns(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
|
|
|
type bounce_type NOT NULL DEFAULT 'hard',
|
|
|
|
source TEXT NOT NULL DEFAULT '',
|
|
|
|
meta JSONB NOT NULL DEFAULT '{}',
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
|
|
|
DROP INDEX IF EXISTS idx_bounces_sub_id; CREATE INDEX idx_bounces_sub_id ON bounces(subscriber_id);
|
|
|
|
DROP INDEX IF EXISTS idx_bounces_camp_id; CREATE INDEX idx_bounces_camp_id ON bounces(campaign_id);
|
|
|
|
DROP INDEX IF EXISTS idx_bounces_source; CREATE INDEX idx_bounces_source ON bounces(source);
|
2021-09-18 18:16:22 +08:00
|
|
|
DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX idx_bounces_date ON bounces((TIMEZONE('UTC', created_at)::DATE));
|
2024-01-12 00:53:39 +08:00
|
|
|
|
2024-06-16 16:20:04 +08:00
|
|
|
-- roles
|
2024-09-02 20:13:56 +08:00
|
|
|
DROP TABLE IF EXISTS roles CASCADE;
|
|
|
|
CREATE TABLE roles (
|
2024-06-16 16:20:04 +08:00
|
|
|
id SERIAL PRIMARY KEY,
|
2024-09-02 20:13:56 +08:00
|
|
|
type role_type NOT NULL DEFAULT 'user',
|
|
|
|
parent_id INTEGER NULL REFERENCES roles(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
2024-06-24 01:20:24 +08:00
|
|
|
list_id INTEGER NULL REFERENCES lists(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
2024-06-16 16:20:04 +08:00
|
|
|
permissions TEXT[] NOT NULL DEFAULT '{}',
|
2024-06-24 01:20:24 +08:00
|
|
|
name TEXT NULL,
|
2024-06-16 16:20:04 +08:00
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
|
|
);
|
2024-10-27 19:48:28 +08:00
|
|
|
CREATE UNIQUE INDEX idx_roles ON roles (parent_id, list_id);
|
|
|
|
CREATE UNIQUE INDEX idx_roles_name ON roles (type, name) WHERE name IS NOT NULL;
|
2024-06-16 16:20:04 +08:00
|
|
|
|
2024-04-03 02:43:57 +08:00
|
|
|
-- users
|
|
|
|
DROP TABLE IF EXISTS users CASCADE;
|
|
|
|
CREATE TABLE users (
|
|
|
|
id SERIAL PRIMARY KEY,
|
|
|
|
username TEXT NOT NULL UNIQUE,
|
|
|
|
password_login BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
password TEXT NULL,
|
|
|
|
email TEXT NOT NULL UNIQUE,
|
|
|
|
name TEXT NOT NULL,
|
2024-07-09 03:11:28 +08:00
|
|
|
avatar TEXT NULL,
|
2024-05-07 13:38:31 +08:00
|
|
|
type user_type NOT NULL DEFAULT 'user',
|
2024-09-02 20:13:56 +08:00
|
|
|
user_role_id INTEGER NOT NULL REFERENCES roles(id) ON DELETE RESTRICT,
|
|
|
|
list_role_id INTEGER NULL REFERENCES roles(id) ON DELETE CASCADE,
|
2024-04-03 02:43:57 +08:00
|
|
|
status user_status NOT NULL DEFAULT 'disabled',
|
|
|
|
loggedin_at TIMESTAMP WITH TIME ZONE NULL,
|
|
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
2024-06-15 17:44:55 +08:00
|
|
|
);
|
|
|
|
|
2024-05-23 14:24:10 +08:00
|
|
|
-- user sessions
|
2024-05-26 02:33:41 +08:00
|
|
|
DROP TABLE IF EXISTS sessions CASCADE;
|
2024-05-23 14:24:10 +08:00
|
|
|
CREATE TABLE sessions (
|
|
|
|
id TEXT NOT NULL PRIMARY KEY,
|
2024-09-02 20:13:56 +08:00
|
|
|
data JSONB DEFAULT '{}'::jsonb NOT NULL,
|
|
|
|
created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now() NOT NULL
|
2024-05-23 14:24:10 +08:00
|
|
|
);
|
2024-05-26 02:33:41 +08:00
|
|
|
DROP INDEX IF EXISTS idx_sessions; CREATE INDEX idx_sessions ON sessions (id, created_at);
|
2024-05-23 14:24:10 +08:00
|
|
|
|
2024-01-12 00:53:39 +08:00
|
|
|
-- materialized views
|
|
|
|
|
|
|
|
-- dashboard stats
|
|
|
|
DROP MATERIALIZED VIEW IF EXISTS mat_dashboard_counts;
|
|
|
|
CREATE MATERIALIZED VIEW mat_dashboard_counts AS
|
|
|
|
WITH subs AS (
|
|
|
|
SELECT COUNT(*) AS num, status FROM subscribers GROUP BY status
|
|
|
|
)
|
|
|
|
SELECT NOW() AS updated_at,
|
|
|
|
JSON_BUILD_OBJECT(
|
|
|
|
'subscribers', JSON_BUILD_OBJECT(
|
|
|
|
'total', (SELECT SUM(num) FROM subs),
|
|
|
|
'blocklisted', (SELECT num FROM subs WHERE status='blocklisted'),
|
|
|
|
'orphans', (
|
|
|
|
SELECT COUNT(id) FROM subscribers
|
|
|
|
LEFT JOIN subscriber_lists ON (subscribers.id = subscriber_lists.subscriber_id)
|
|
|
|
WHERE subscriber_lists.subscriber_id IS NULL
|
|
|
|
)
|
|
|
|
),
|
|
|
|
'lists', JSON_BUILD_OBJECT(
|
|
|
|
'total', (SELECT COUNT(*) FROM lists),
|
|
|
|
'private', (SELECT COUNT(*) FROM lists WHERE type='private'),
|
|
|
|
'public', (SELECT COUNT(*) FROM lists WHERE type='public'),
|
|
|
|
'optin_single', (SELECT COUNT(*) FROM lists WHERE optin='single'),
|
|
|
|
'optin_double', (SELECT COUNT(*) FROM lists WHERE optin='double')
|
|
|
|
),
|
|
|
|
'campaigns', JSON_BUILD_OBJECT(
|
|
|
|
'total', (SELECT COUNT(*) FROM campaigns),
|
|
|
|
'by_status', (
|
|
|
|
SELECT JSON_OBJECT_AGG (status, num) FROM
|
|
|
|
(SELECT status, COUNT(*) AS num FROM campaigns GROUP BY status) r
|
|
|
|
)
|
|
|
|
),
|
|
|
|
'messages', (SELECT SUM(sent) AS messages FROM campaigns)
|
|
|
|
) AS data;
|
|
|
|
DROP INDEX IF EXISTS mat_dashboard_stats_idx; CREATE UNIQUE INDEX mat_dashboard_stats_idx ON mat_dashboard_counts (updated_at);
|
|
|
|
|
|
|
|
|
|
|
|
DROP MATERIALIZED VIEW IF EXISTS mat_dashboard_charts;
|
|
|
|
CREATE MATERIALIZED VIEW mat_dashboard_charts AS
|
|
|
|
WITH clicks AS (
|
|
|
|
SELECT JSON_AGG(ROW_TO_JSON(row))
|
|
|
|
FROM (
|
|
|
|
WITH viewDates AS (
|
|
|
|
SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,
|
|
|
|
TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date
|
|
|
|
FROM link_clicks ORDER BY id DESC LIMIT 1
|
|
|
|
)
|
|
|
|
SELECT COUNT(*) AS count, created_at::DATE as date FROM link_clicks
|
|
|
|
-- use > between < to force the use of the date index.
|
|
|
|
WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)
|
|
|
|
GROUP by date ORDER BY date
|
|
|
|
) row
|
|
|
|
),
|
|
|
|
views AS (
|
|
|
|
SELECT JSON_AGG(ROW_TO_JSON(row))
|
|
|
|
FROM (
|
|
|
|
WITH viewDates AS (
|
|
|
|
SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,
|
|
|
|
TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date
|
|
|
|
FROM campaign_views ORDER BY id DESC LIMIT 1
|
|
|
|
)
|
|
|
|
SELECT COUNT(*) AS count, created_at::DATE as date FROM campaign_views
|
|
|
|
-- use > between < to force the use of the date index.
|
|
|
|
WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)
|
|
|
|
GROUP by date ORDER BY date
|
|
|
|
) row
|
|
|
|
)
|
|
|
|
SELECT NOW() AS updated_at, JSON_BUILD_OBJECT('link_clicks', COALESCE((SELECT * FROM clicks), '[]'),
|
|
|
|
'campaign_views', COALESCE((SELECT * FROM views), '[]')
|
|
|
|
) AS data;
|
|
|
|
DROP INDEX IF EXISTS mat_dashboard_charts_idx; CREATE UNIQUE INDEX mat_dashboard_charts_idx ON mat_dashboard_charts (updated_at);
|
|
|
|
|
|
|
|
-- subscriber counts stats for lists
|
|
|
|
DROP MATERIALIZED VIEW IF EXISTS mat_list_subscriber_stats;
|
|
|
|
CREATE MATERIALIZED VIEW mat_list_subscriber_stats AS
|
|
|
|
SELECT NOW() AS updated_at, lists.id AS list_id, subscriber_lists.status, COUNT(*) AS subscriber_count FROM lists
|
|
|
|
LEFT JOIN subscriber_lists ON (subscriber_lists.list_id = lists.id)
|
|
|
|
GROUP BY lists.id, subscriber_lists.status
|
|
|
|
UNION ALL
|
|
|
|
SELECT NOW() AS updated_at, 0 AS list_id, NULL AS status, COUNT(*) AS subscriber_count FROM subscribers;
|
|
|
|
DROP INDEX IF EXISTS mat_list_subscriber_stats_idx; CREATE UNIQUE INDEX mat_list_subscriber_stats_idx ON mat_list_subscriber_stats (list_id, status);
|