From 5a3664aee29ee44efacf42dbcab2a5e79ebfe835 Mon Sep 17 00:00:00 2001 From: Kailash Nadh Date: Thu, 11 Jan 2024 22:23:39 +0530 Subject: [PATCH] Add support for caching slow queries on large databases. - Add materialized views for list -> subscriber counts, dashboard chart, and dashboard aggregate stats that slow down significantly on large databases (with millions or tens of millions of subscribers). These slow queries involve full table scan COUNTS(). - Add a toggle to enable caching slow results in Settings -> Performance. - Add support for setting a cron string that crons and periodically refreshes aggregated stats in materialized views. Closes #1019. --- cmd/init.go | 23 +++- cmd/main.go | 8 +- cmd/settings.go | 8 ++ cmd/upgrade.go | 5 +- frontend/cypress/e2e/subscribers.cy.js | 8 +- frontend/src/views/Campaign.vue | 3 +- frontend/src/views/Dashboard.vue | 9 ++ frontend/src/views/Lists.vue | 8 ++ frontend/src/views/settings/performance.vue | 24 ++++ go.mod | 1 + go.sum | 2 + i18n/ca.json | 4 + i18n/cs-cz.json | 4 + i18n/cy.json | 4 + i18n/da.json | 4 + i18n/de.json | 4 + i18n/el.json | 4 + i18n/en.json | 4 + i18n/es.json | 4 + i18n/fi.json | 4 + i18n/fr.json | 4 + i18n/he.json | 4 + i18n/hu.json | 4 + i18n/it.json | 4 + i18n/jp.json | 4 + i18n/ml.json | 4 + i18n/nl.json | 4 + i18n/pl.json | 4 + i18n/pt-BR.json | 4 + i18n/pt.json | 4 + i18n/ro.json | 4 + i18n/ru.json | 4 + i18n/se.json | 4 + i18n/sk.json | 4 + i18n/tr.json | 4 + i18n/uk.json | 4 + i18n/vi.json | 4 + i18n/zh-CN.json | 4 + i18n/zh-TW.json | 4 + internal/core/bounces.go | 2 +- internal/core/core.go | 61 +++++++++-- internal/core/dashboard.go | 4 + internal/core/lists.go | 8 +- internal/core/subscribers.go | 61 ++++++++--- internal/migrations/v0.4.0.go | 4 +- internal/migrations/v0.7.0.go | 4 +- internal/migrations/v0.8.0.go | 4 +- internal/migrations/v0.9.0.go | 3 +- internal/migrations/v1.0.0.go | 4 +- internal/migrations/v2.0.0.go | 4 +- internal/migrations/v2.1.0.go | 4 +- internal/migrations/v2.2.0.go | 4 +- internal/migrations/v2.3.0.go | 4 +- internal/migrations/v2.4.0.go | 4 +- internal/migrations/v2.5.0.go | 4 +- internal/migrations/v3.0.0.go | 115 +++++++++++++++++++- models/queries.go | 19 ++-- models/settings.go | 10 +- queries.sql | 89 +++------------ schema.sql | 100 +++++++++++++++++ 60 files changed, 587 insertions(+), 136 deletions(-) diff --git a/cmd/init.go b/cmd/init.go index 56cb6c48..03c9f475 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Masterminds/sprig/v3" + "github.com/gdgvda/cron" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/types" "github.com/knadh/goyesql/v2" @@ -28,6 +29,7 @@ import ( "github.com/knadh/listmonk/internal/bounce" "github.com/knadh/listmonk/internal/bounce/mailbox" "github.com/knadh/listmonk/internal/captcha" + "github.com/knadh/listmonk/internal/core" "github.com/knadh/listmonk/internal/i18n" "github.com/knadh/listmonk/internal/manager" "github.com/knadh/listmonk/internal/media" @@ -494,7 +496,7 @@ func initTxTemplates(m *manager.Manager, app *App) { } // initImporter initializes the bulk subscriber importer. -func initImporter(q *models.Queries, db *sqlx.DB, app *App) *subimporter.Importer { +func initImporter(q *models.Queries, db *sqlx.DB, core *core.Core, app *App) *subimporter.Importer { return subimporter.New( subimporter.Options{ DomainBlocklist: app.constants.Privacy.DomainBlocklist, @@ -502,6 +504,9 @@ func initImporter(q *models.Queries, db *sqlx.DB, app *App) *subimporter.Importe BlocklistStmt: q.UpsertBlocklistSubscriber.Stmt, UpdateListDateStmt: q.UpdateListsDate.Stmt, NotifCB: func(subject string, data interface{}) error { + // Refresh cached subscriber counts and stats. + core.RefreshMatViews(true) + app.sendNotification(app.constants.NotifyEmails, subject, notifTplImport, data) return nil }, @@ -803,6 +808,22 @@ func initCaptcha() *captcha.Captcha { }) } +func initCron(core *core.Core) { + c := cron.New() + _, err := c.Add(ko.MustString("app.cache_slow_queries_interval"), func() { + lo.Println("refreshing slow query cache") + _ = core.RefreshMatViews(true) + lo.Println("done refreshing slow query cache") + }) + if err != nil { + lo.Printf("error initializing slow cache query cron: %v", err) + return + } + + c.Start() + lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[0].Next) +} + func awaitReload(sigChan chan os.Signal, closerWait chan bool, closer func()) chan bool { // The blocking signal handler that main() waits on. out := make(chan bool) diff --git a/cmd/main.go b/cmd/main.go index 26d69ecc..2d51f205 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -191,6 +191,7 @@ func main() { cOpt := &core.Opt{ Constants: core.Constants{ SendOptinConfirmation: app.constants.SendOptinConfirmation, + CacheSlowQueries: ko.Bool("app.cache_slow_queries"), }, Queries: queries, DB: db, @@ -208,7 +209,7 @@ func main() { app.queries = queries app.manager = initCampaignManager(app.queries, app.constants, app) - app.importer = initImporter(app.queries, db, app) + app.importer = initImporter(app.queries, db, app.core, app) app.notifTpls = initNotifTemplates("/email-templates/*.html", fs, app.i18n, app.constants) initTxTemplates(app.manager, app) @@ -233,6 +234,11 @@ func main() { // Load system information. app.about = initAbout(queries, db) + // Start cronjobs. + if cOpt.Constants.CacheSlowQueries { + initCron(app.core) + } + // Start the campaign workers. The campaign batches (fetch from DB, push out // messages) get processed at the specified interval. go app.manager.Run() diff --git a/cmd/settings.go b/cmd/settings.go index 7c0f0d57..dc7a5b45 100644 --- a/cmd/settings.go +++ b/cmd/settings.go @@ -11,6 +11,7 @@ import ( "time" "unicode/utf8" + "github.com/gdgvda/cron" "github.com/gofrs/uuid" "github.com/jmoiron/sqlx/types" "github.com/knadh/koanf/parsers/json" @@ -207,6 +208,13 @@ func handleUpdateSettings(c echo.Context) error { } set.DomainBlocklist = doms + // Validate slow query caching cron. + if set.CacheSlowQueries { + if _, err := cron.ParseStandard(set.CacheSlowQueriesInterval); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("globals.messages.invalidData")+": slow query cron: "+err.Error()) + } + } + // Update the settings in the DB. if err := app.core.UpdateSettings(set); err != nil { return err diff --git a/cmd/upgrade.go b/cmd/upgrade.go index e3c562a5..011ebbfd 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "strings" "github.com/jmoiron/sqlx" @@ -18,7 +19,7 @@ import ( // of logic to be performed before executing upgrades. fn is idempotent. type migFunc struct { version string - fn func(*sqlx.DB, stuffbin.FileSystem, *koanf.Koanf) error + fn func(*sqlx.DB, stuffbin.FileSystem, *koanf.Koanf, *log.Logger) error } // migList is the list of available migList ordered by the semver. @@ -69,7 +70,7 @@ func upgrade(db *sqlx.DB, fs stuffbin.FileSystem, prompt bool) { // Execute migrations in succession. for _, m := range toRun { lo.Printf("running migration %s", m.version) - if err := m.fn(db, fs, ko); err != nil { + if err := m.fn(db, fs, ko, lo); err != nil { lo.Fatalf("error running migration %s: %v", m.version, err) } diff --git a/frontend/cypress/e2e/subscribers.cy.js b/frontend/cypress/e2e/subscribers.cy.js index 338bcea7..ad8ea44b 100644 --- a/frontend/cypress/e2e/subscribers.cy.js +++ b/frontend/cypress/e2e/subscribers.cy.js @@ -147,7 +147,7 @@ describe('Subscribers', () => { // Get the ID from the header and proceed to fill the form. let id = 0; cy.get('[data-cy=id]').then(($el) => { - id = $el.text(); + id = parseInt($el.text()); cy.get('input[name=email]').clear().type(email); cy.get('input[name=name]').clear().type(name); @@ -162,9 +162,11 @@ describe('Subscribers', () => { }); // Confirm the edits on the table. - cy.wait(250); + cy.wait(500); + cy.log(rows); cy.get('tbody tr').each(($el) => { - cy.wrap($el).find('td[data-id]').invoke('attr', 'data-id').then((id) => { + cy.wrap($el).find('td[data-id]').invoke('attr', 'data-id').then((idStr) => { + const id = parseInt(idStr); cy.wrap($el).find('td[data-label=E-mail]').contains(rows[id].email.toLowerCase()); cy.wrap($el).find('td[data-label=Name]').contains(rows[id].name); cy.wrap($el).find('td[data-label=Status]').contains(rows[id].status, { matchCase: false }); diff --git a/frontend/src/views/Campaign.vue b/frontend/src/views/Campaign.vue index a6804fa6..b1309890 100644 --- a/frontend/src/views/Campaign.vue +++ b/frontend/src/views/Campaign.vue @@ -183,7 +183,7 @@
- Templating reference + {{ $t('campaigns.templatingRef') }} {{ $t('campaigns.addAltText') }} @@ -193,7 +193,6 @@ {{ $t('campaigns.removeAltText') }} -
diff --git a/frontend/src/views/Dashboard.vue b/frontend/src/views/Dashboard.vue index dbf52f5e..e279e4c6 100644 --- a/frontend/src/views/Dashboard.vue +++ b/frontend/src/views/Dashboard.vue @@ -137,6 +137,13 @@ +

+ *{{ $t('globals.messages.slowQueriesCached') }} + + {{ $t('globals.buttons.learnMore') }} + +

@@ -144,6 +151,7 @@