2022-04-03 23:24:40 +08:00
|
|
|
package models
|
2018-10-25 21:51:47 +08:00
|
|
|
|
|
|
|
import (
|
2018-12-18 13:24:55 +08:00
|
|
|
"context"
|
|
|
|
"database/sql"
|
2018-10-25 21:51:47 +08:00
|
|
|
"fmt"
|
|
|
|
|
|
|
|
"github.com/jmoiron/sqlx"
|
2019-03-09 15:46:47 +08:00
|
|
|
"github.com/lib/pq"
|
2018-10-25 21:51:47 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
// Queries contains all prepared SQL queries.
|
|
|
|
type Queries struct {
|
2020-07-05 00:55:02 +08:00
|
|
|
GetDashboardCharts *sqlx.Stmt `query:"get-dashboard-charts"`
|
|
|
|
GetDashboardCounts *sqlx.Stmt `query:"get-dashboard-counts"`
|
2018-11-05 13:49:08 +08:00
|
|
|
|
2018-12-18 13:24:55 +08:00
|
|
|
InsertSubscriber *sqlx.Stmt `query:"insert-subscriber"`
|
|
|
|
UpsertSubscriber *sqlx.Stmt `query:"upsert-subscriber"`
|
2020-08-01 19:15:29 +08:00
|
|
|
UpsertBlocklistSubscriber *sqlx.Stmt `query:"upsert-blocklist-subscriber"`
|
2018-12-18 13:24:55 +08:00
|
|
|
GetSubscriber *sqlx.Stmt `query:"get-subscriber"`
|
2024-09-02 20:13:56 +08:00
|
|
|
HasSubscriberLists *sqlx.Stmt `query:"has-subscriber-list"`
|
2018-12-18 13:24:55 +08:00
|
|
|
GetSubscribersByEmails *sqlx.Stmt `query:"get-subscribers-by-emails"`
|
|
|
|
GetSubscriberLists *sqlx.Stmt `query:"get-subscriber-lists"`
|
2022-10-19 00:14:57 +08:00
|
|
|
GetSubscriptions *sqlx.Stmt `query:"get-subscriptions"`
|
2019-12-01 20:18:36 +08:00
|
|
|
GetSubscriberListsLazy *sqlx.Stmt `query:"get-subscriber-lists-lazy"`
|
2018-12-18 13:24:55 +08:00
|
|
|
UpdateSubscriber *sqlx.Stmt `query:"update-subscriber"`
|
2022-10-19 00:14:57 +08:00
|
|
|
UpdateSubscriberWithLists *sqlx.Stmt `query:"update-subscriber-with-lists"`
|
2020-08-01 19:15:29 +08:00
|
|
|
BlocklistSubscribers *sqlx.Stmt `query:"blocklist-subscribers"`
|
2018-12-18 13:24:55 +08:00
|
|
|
AddSubscribersToLists *sqlx.Stmt `query:"add-subscribers-to-lists"`
|
|
|
|
DeleteSubscriptions *sqlx.Stmt `query:"delete-subscriptions"`
|
2022-09-03 15:48:02 +08:00
|
|
|
DeleteUnconfirmedSubscriptions *sqlx.Stmt `query:"delete-unconfirmed-subscriptions"`
|
2019-12-01 20:18:36 +08:00
|
|
|
ConfirmSubscriptionOptin *sqlx.Stmt `query:"confirm-subscription-optin"`
|
2018-12-18 13:24:55 +08:00
|
|
|
UnsubscribeSubscribersFromLists *sqlx.Stmt `query:"unsubscribe-subscribers-from-lists"`
|
|
|
|
DeleteSubscribers *sqlx.Stmt `query:"delete-subscribers"`
|
2022-09-03 15:48:02 +08:00
|
|
|
DeleteBlocklistedSubscribers *sqlx.Stmt `query:"delete-blocklisted-subscribers"`
|
|
|
|
DeleteOrphanSubscribers *sqlx.Stmt `query:"delete-orphan-subscribers"`
|
2022-05-08 17:15:45 +08:00
|
|
|
UnsubscribeByCampaign *sqlx.Stmt `query:"unsubscribe-by-campaign"`
|
2019-07-21 21:48:41 +08:00
|
|
|
ExportSubscriberData *sqlx.Stmt `query:"export-subscriber-data"`
|
2018-12-18 13:24:55 +08:00
|
|
|
|
|
|
|
// Non-prepared arbitrary subscriber queries.
|
2024-01-12 00:53:39 +08:00
|
|
|
QuerySubscribers string `query:"query-subscribers"`
|
|
|
|
QuerySubscribersCount string `query:"query-subscribers-count"`
|
|
|
|
QuerySubscribersCountAll *sqlx.Stmt `query:"query-subscribers-count-all"`
|
|
|
|
QuerySubscribersForExport string `query:"query-subscribers-for-export"`
|
|
|
|
QuerySubscribersTpl string `query:"query-subscribers-template"`
|
|
|
|
DeleteSubscribersByQuery string `query:"delete-subscribers-by-query"`
|
|
|
|
AddSubscribersToListsByQuery string `query:"add-subscribers-to-lists-by-query"`
|
|
|
|
BlocklistSubscribersByQuery string `query:"blocklist-subscribers-by-query"`
|
|
|
|
DeleteSubscriptionsByQuery string `query:"delete-subscriptions-by-query"`
|
|
|
|
UnsubscribeSubscribersFromListsByQuery string `query:"unsubscribe-subscribers-from-lists-by-query"`
|
2018-10-25 21:51:47 +08:00
|
|
|
|
2019-05-15 00:36:14 +08:00
|
|
|
CreateList *sqlx.Stmt `query:"create-list"`
|
2021-01-31 18:49:39 +08:00
|
|
|
QueryLists string `query:"query-lists"`
|
|
|
|
GetLists *sqlx.Stmt `query:"get-lists"`
|
2019-12-01 20:18:36 +08:00
|
|
|
GetListsByOptin *sqlx.Stmt `query:"get-lists-by-optin"`
|
2019-05-15 00:36:14 +08:00
|
|
|
UpdateList *sqlx.Stmt `query:"update-list"`
|
|
|
|
UpdateListsDate *sqlx.Stmt `query:"update-lists-date"`
|
|
|
|
DeleteLists *sqlx.Stmt `query:"delete-lists"`
|
2018-10-25 21:51:47 +08:00
|
|
|
|
2022-07-30 21:31:20 +08:00
|
|
|
CreateCampaign *sqlx.Stmt `query:"create-campaign"`
|
|
|
|
QueryCampaigns string `query:"query-campaigns"`
|
|
|
|
GetCampaign *sqlx.Stmt `query:"get-campaign"`
|
|
|
|
GetCampaignForPreview *sqlx.Stmt `query:"get-campaign-for-preview"`
|
|
|
|
GetCampaignStats *sqlx.Stmt `query:"get-campaign-stats"`
|
|
|
|
GetCampaignStatus *sqlx.Stmt `query:"get-campaign-status"`
|
2022-11-03 13:37:26 +08:00
|
|
|
GetArchivedCampaigns *sqlx.Stmt `query:"get-archived-campaigns"`
|
2022-07-30 21:31:20 +08:00
|
|
|
|
|
|
|
// These two queries are read as strings and based on settings.individual_tracking=on/off,
|
|
|
|
// are interpolated and copied to view and click counts. Same query, different tables.
|
2023-12-27 01:40:33 +08:00
|
|
|
GetCampaignAnalyticsCounts string `query:"get-campaign-analytics-counts"`
|
|
|
|
GetCampaignViewCounts *sqlx.Stmt `query:"get-campaign-view-counts"`
|
|
|
|
GetCampaignClickCounts *sqlx.Stmt `query:"get-campaign-click-counts"`
|
|
|
|
GetCampaignLinkCounts *sqlx.Stmt `query:"get-campaign-link-counts"`
|
|
|
|
GetCampaignBounceCounts *sqlx.Stmt `query:"get-campaign-bounce-counts"`
|
|
|
|
DeleteCampaignViews *sqlx.Stmt `query:"delete-campaign-views"`
|
|
|
|
DeleteCampaignLinkClicks *sqlx.Stmt `query:"delete-campaign-link-clicks"`
|
2022-07-30 21:31:20 +08:00
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
NextCampaigns *sqlx.Stmt `query:"next-campaigns"`
|
Fix and refactor subscriber batch fetching in campaign processing.
This has been a hair-pulling rabbit hole of an issue. #1931 and others.
When the `next-campaign-subscribers` query that fetches $n subscribers
per batch for a campaign returns no results, the manager assumes
that the campaign is done and marks as finished.
Marathon debugging revealed fundamental flaws in qyery's logic that
would incorrectly return 0 rows under certain conditions.
- Based on the "layout" of subscribers for eg: a series of blocklisted
subscribers between confirmed subscribers.
A series of unconfirmed subscribers in a batch belonging to a double
opt-in list.
- Bulk import blocklisting users, but not marking their subscriptions
as 'unsubscribed'.
- Conditions spread across multiple CTEs resulted in returning an
arbitrary number of rows and $N per batch as the selected $N rows
would get filtered out elsewhere, possibly even becoming 0.
After fixing this and testing it on our prod instance that has
15 million subscribers and ~70 million subscriptions in the
`subscriber_lists` table, ended up discovered significant inefficiences
in Postgres query planning. When `subscriber_lists` and campaign list IDs
are joined dynamically (CTE or ANY() or any kind of JOIN that involves)
a query, the Postgres query planner is unable to use the right indexes.
After testing dozens of approaches, discovered that statically passing
the values to join on (hardcoding or passing via parametrized $1 vars),
the query uses the right indexes. The difference is staggering.
For the particular scenario on our large prod DB to pull a batch,
~15 seconds vs. ~50ms, a whopping 300x improvement!
This patch splits `next-campaign-subscribers` into two separate queries,
one which fetches campaign metadata and list_ids, whose values are then
passed statically to the next query to fetch subscribers by batch.
In addition, it fixes and refactors broken filtering and counting logic
in `create-campaign` and `next-campaign` queries.
Closes #1931, #1993, #1986.
2024-09-20 20:21:44 +08:00
|
|
|
GetRunningCampaign *sqlx.Stmt `query:"get-running-campaign"`
|
2018-10-25 21:51:47 +08:00
|
|
|
NextCampaignSubscribers *sqlx.Stmt `query:"next-campaign-subscribers"`
|
|
|
|
GetOneCampaignSubscriber *sqlx.Stmt `query:"get-one-campaign-subscriber"`
|
|
|
|
UpdateCampaign *sqlx.Stmt `query:"update-campaign"`
|
|
|
|
UpdateCampaignStatus *sqlx.Stmt `query:"update-campaign-status"`
|
|
|
|
UpdateCampaignCounts *sqlx.Stmt `query:"update-campaign-counts"`
|
2022-11-03 13:37:26 +08:00
|
|
|
UpdateCampaignArchive *sqlx.Stmt `query:"update-campaign-archive"`
|
2018-11-02 15:50:32 +08:00
|
|
|
RegisterCampaignView *sqlx.Stmt `query:"register-campaign-view"`
|
2018-10-25 21:51:47 +08:00
|
|
|
DeleteCampaign *sqlx.Stmt `query:"delete-campaign"`
|
|
|
|
|
|
|
|
InsertMedia *sqlx.Stmt `query:"insert-media"`
|
|
|
|
GetMedia *sqlx.Stmt `query:"get-media"`
|
2023-05-21 17:49:12 +08:00
|
|
|
QueryMedia *sqlx.Stmt `query:"query-media"`
|
2018-10-25 21:51:47 +08:00
|
|
|
DeleteMedia *sqlx.Stmt `query:"delete-media"`
|
|
|
|
|
|
|
|
CreateTemplate *sqlx.Stmt `query:"create-template"`
|
|
|
|
GetTemplates *sqlx.Stmt `query:"get-templates"`
|
|
|
|
UpdateTemplate *sqlx.Stmt `query:"update-template"`
|
|
|
|
SetDefaultTemplate *sqlx.Stmt `query:"set-default-template"`
|
|
|
|
DeleteTemplate *sqlx.Stmt `query:"delete-template"`
|
|
|
|
|
2018-10-31 20:54:21 +08:00
|
|
|
CreateLink *sqlx.Stmt `query:"create-link"`
|
|
|
|
RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`
|
|
|
|
|
2020-07-08 19:00:14 +08:00
|
|
|
GetSettings *sqlx.Stmt `query:"get-settings"`
|
|
|
|
UpdateSettings *sqlx.Stmt `query:"update-settings"`
|
|
|
|
|
2018-10-25 21:51:47 +08:00
|
|
|
// GetStats *sqlx.Stmt `query:"get-stats"`
|
2021-05-25 01:11:48 +08:00
|
|
|
RecordBounce *sqlx.Stmt `query:"record-bounce"`
|
|
|
|
QueryBounces string `query:"query-bounces"`
|
|
|
|
DeleteBounces *sqlx.Stmt `query:"delete-bounces"`
|
|
|
|
DeleteBouncesBySubscriber *sqlx.Stmt `query:"delete-bounces-by-subscriber"`
|
2023-06-24 15:37:13 +08:00
|
|
|
GetDBInfo string `query:"get-db-info"`
|
2024-04-03 02:43:57 +08:00
|
|
|
|
2024-05-31 02:07:20 +08:00
|
|
|
CreateUser *sqlx.Stmt `query:"create-user"`
|
|
|
|
UpdateUser *sqlx.Stmt `query:"update-user"`
|
|
|
|
UpdateUserProfile *sqlx.Stmt `query:"update-user-profile"`
|
2024-07-09 03:12:29 +08:00
|
|
|
UpdateUserLogin *sqlx.Stmt `query:"update-user-login"`
|
2024-05-31 02:07:20 +08:00
|
|
|
DeleteUsers *sqlx.Stmt `query:"delete-users"`
|
|
|
|
GetUsers *sqlx.Stmt `query:"get-users"`
|
2024-10-26 19:33:02 +08:00
|
|
|
GetUser *sqlx.Stmt `query:"get-user"`
|
2024-05-31 02:07:20 +08:00
|
|
|
GetAPITokens *sqlx.Stmt `query:"get-api-tokens"`
|
|
|
|
LoginUser *sqlx.Stmt `query:"login-user"`
|
2024-06-15 17:44:55 +08:00
|
|
|
|
2024-06-24 01:20:24 +08:00
|
|
|
CreateRole *sqlx.Stmt `query:"create-role"`
|
2024-09-02 20:13:56 +08:00
|
|
|
GetUserRoles *sqlx.Stmt `query:"get-user-roles"`
|
|
|
|
GetListRoles *sqlx.Stmt `query:"get-list-roles"`
|
2024-06-24 01:20:24 +08:00
|
|
|
UpdateRole *sqlx.Stmt `query:"update-role"`
|
|
|
|
DeleteRole *sqlx.Stmt `query:"delete-role"`
|
|
|
|
UpsertListPermissions *sqlx.Stmt `query:"upsert-list-permissions"`
|
|
|
|
DeleteListPermission *sqlx.Stmt `query:"delete-list-permission"`
|
2018-10-25 21:51:47 +08:00
|
|
|
}
|
|
|
|
|
2022-04-03 23:24:40 +08:00
|
|
|
// CompileSubscriberQueryTpl takes an arbitrary WHERE expressions
|
2018-12-18 13:24:55 +08:00
|
|
|
// to filter subscribers from the subscribers table and prepares a query
|
|
|
|
// out of it using the raw `query-subscribers-template` query template.
|
|
|
|
// While doing this, a readonly transaction is created and the query is
|
|
|
|
// dry run on it to ensure that it is indeed readonly.
|
2024-09-19 13:26:56 +08:00
|
|
|
func (q *Queries) CompileSubscriberQueryTpl(exp string, db *sqlx.DB, subStatus string) (string, error) {
|
2018-12-18 13:24:55 +08:00
|
|
|
tx, err := db.BeginTxx(context.Background(), &sql.TxOptions{ReadOnly: true})
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2019-12-05 23:57:31 +08:00
|
|
|
defer tx.Rollback()
|
2018-12-18 13:24:55 +08:00
|
|
|
|
|
|
|
// Perform the dry run.
|
|
|
|
if exp != "" {
|
|
|
|
exp = " AND " + exp
|
|
|
|
}
|
|
|
|
stmt := fmt.Sprintf(q.QuerySubscribersTpl, exp)
|
2024-09-19 13:26:56 +08:00
|
|
|
if _, err := tx.Exec(stmt, true, pq.Int64Array{}, subStatus); err != nil {
|
2018-12-18 13:24:55 +08:00
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return stmt, nil
|
|
|
|
}
|
|
|
|
|
2022-02-28 21:19:50 +08:00
|
|
|
// compileSubscriberQueryTpl takes an arbitrary WHERE expressions and a subscriber
|
2020-08-01 19:15:29 +08:00
|
|
|
// query template that depends on the filter (eg: delete by query, blocklist by query etc.)
|
2018-12-18 13:24:55 +08:00
|
|
|
// combines and executes them.
|
2024-09-19 13:26:56 +08:00
|
|
|
func (q *Queries) ExecSubQueryTpl(exp, tpl string, listIDs []int, db *sqlx.DB, subStatus string, args ...interface{}) error {
|
2018-12-18 13:24:55 +08:00
|
|
|
// Perform a dry run.
|
2024-09-19 13:26:56 +08:00
|
|
|
filterExp, err := q.CompileSubscriberQueryTpl(exp, db, subStatus)
|
2018-12-18 13:24:55 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(listIDs) == 0 {
|
2022-04-03 23:24:40 +08:00
|
|
|
listIDs = []int{}
|
2018-12-18 13:24:55 +08:00
|
|
|
}
|
2022-03-20 13:32:43 +08:00
|
|
|
|
2018-12-18 13:24:55 +08:00
|
|
|
// First argument is the boolean indicating if the query is a dry run.
|
2024-09-19 13:26:56 +08:00
|
|
|
a := append([]interface{}{false, pq.Array(listIDs), subStatus}, args...)
|
2018-12-18 13:24:55 +08:00
|
|
|
if _, err := db.Exec(fmt.Sprintf(tpl, filterExp), a...); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|