package models import ( "bytes" "database/sql/driver" "encoding/json" "errors" "fmt" "html/template" "net/textproto" "regexp" "strings" txttpl "text/template" "time" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/types" "github.com/lib/pq" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/renderer/html" null "gopkg.in/volatiletech/null.v6" ) // Enum values for various statuses. const ( // Subscriber. SubscriberStatusEnabled = "enabled" SubscriberStatusDisabled = "disabled" SubscriberStatusBlockListed = "blocklisted" // Subscription. SubscriptionStatusUnconfirmed = "unconfirmed" SubscriptionStatusConfirmed = "confirmed" SubscriptionStatusUnsubscribed = "unsubscribed" // Campaign. CampaignStatusDraft = "draft" CampaignStatusScheduled = "scheduled" CampaignStatusRunning = "running" CampaignStatusPaused = "paused" CampaignStatusFinished = "finished" CampaignStatusCancelled = "cancelled" CampaignTypeRegular = "regular" CampaignTypeOptin = "optin" CampaignContentTypeRichtext = "richtext" CampaignContentTypeHTML = "html" CampaignContentTypeMarkdown = "markdown" CampaignContentTypePlain = "plain" // List. ListTypePrivate = "private" ListTypePublic = "public" ListOptinSingle = "single" ListOptinDouble = "double" // User. UserTypeUser = "user" UserTypeAPI = "api" UserStatusEnabled = "enabled" UserStatusDisabled = "disabled" // BaseTpl is the name of the base template. BaseTpl = "base" // ContentTpl is the name of the compiled message. ContentTpl = "content" // Headers attached to e-mails for bounce tracking. EmailHeaderSubscriberUUID = "X-Listmonk-Subscriber" EmailHeaderCampaignUUID = "X-Listmonk-Campaign" // Standard e-mail headers. EmailHeaderDate = "Date" EmailHeaderFrom = "From" EmailHeaderSubject = "Subject" EmailHeaderMessageId = "Message-Id" EmailHeaderDeliveredTo = "Delivered-To" EmailHeaderReceived = "Received" BounceTypeHard = "hard" BounceTypeSoft = "soft" BounceTypeComplaint = "complaint" // Templates. TemplateTypeCampaign = "campaign" TemplateTypeTx = "tx" ) // Headers represents an array of string maps used to represent SMTP, HTTP headers etc. // similar to url.Values{} type Headers []map[string]string // regTplFunc represents contains a regular expression for wrapping and // substituting a Go template function from the user's shorthand to a full // function call. type regTplFunc struct { regExp *regexp.Regexp replace string } var regTplFuncs = []regTplFunc{ // Regular expression for matching {{ TrackLink "http://link.com" }} in the template // and substituting it with {{ Track "http://link.com" . }} (the dot context) // before compilation. This is to make linking easier for users. { regExp: regexp.MustCompile("{{(\\s+)?TrackLink(\\s+)?(.+?)(\\s+)?}}"), replace: `{{ TrackLink $3 . }}`, }, // Convert the shorthand https://google.com@TrackLink to {{ TrackLink ... }}. // This is for WYSIWYG editors that encode and break quotes {{ "" }} when inserted // inside . { regExp: regexp.MustCompile(`(https?://.+?)@TrackLink`), replace: `{{ TrackLink "$1" . }}`, }, { regExp: regexp.MustCompile(`{{(\s+)?(TrackView|UnsubscribeURL|ManageURL|OptinURL|MessageURL)(\s+)?}}`), replace: `{{ $2 . }}`, }, } // AdminNotifCallback is a callback function that's called // when a campaign's status changes. type AdminNotifCallback func(subject string, data interface{}) error // PageResults is a generic HTTP response container for paginated results of list of items. type PageResults struct { Results interface{} `json:"results"` Query string `json:"query"` Total int `json:"total"` PerPage int `json:"per_page"` Page int `json:"page"` } // Base holds common fields shared across models. type Base struct { ID int `db:"id" json:"id"` CreatedAt null.Time `db:"created_at" json:"created_at"` UpdatedAt null.Time `db:"updated_at" json:"updated_at"` } // User represents an admin user. type User struct { Base Username string `db:"username" json:"username"` // For API users, this is the plaintext API token. Password null.String `db:"password" json:"password,omitempty"` PasswordLogin bool `db:"password_login" json:"password_login"` Email null.String `db:"email" json:"email"` Name string `db:"name" json:"name"` Type string `db:"type" json:"type"` Status string `db:"status" json:"status"` Avatar null.String `db:"-" json:"avatar"` LoggedInAt null.Time `db:"loggedin_at" json:"loggedin_at"` // Filled post-retrieval. Role struct { ID int `db:"-" json:"id"` Name string `db:"-" json:"name"` Permissions []string `db:"-" json:"permissions"` Lists []ListPermission `db:"-" json:"lists"` } `db:"-" json:"role"` RoleID int `db:"role_id" json:"role_id,omitempty"` RoleName string `db:"role_name" json:"-"` RolePerms pq.StringArray `db:"role_permissions" json:"-"` ListsPermsRaw json.RawMessage `db:"list_permissions" json:"-"` PermissionsMap map[string]struct{} `db:"-" json:"-"` ListPermissionsMap map[int]map[string]struct{} `db:"-" json:"-"` HasPassword bool `db:"-" json:"-"` } type ListPermission struct { ID int `json:"id"` Name string `json:"name"` Permissions pq.StringArray `json:"permissions"` } type Role struct { Base Name null.String `db:"name" json:"name"` Permissions pq.StringArray `db:"permissions" json:"permissions"` ListID null.Int `db:"list_id" json:"-"` ParentID null.Int `db:"parent_id" json:"-"` ListsRaw json.RawMessage `db:"list_permissions" json:"-"` Lists []ListPermission `db:"-" json:"lists"` } // Subscriber represents an e-mail subscriber. type Subscriber struct { Base UUID string `db:"uuid" json:"uuid"` Email string `db:"email" json:"email" form:"email"` Name string `db:"name" json:"name" form:"name"` Attribs JSON `db:"attribs" json:"attribs"` Status string `db:"status" json:"status"` Lists types.JSONText `db:"lists" json:"lists"` } type subLists struct { SubscriberID int `db:"subscriber_id"` Lists types.JSONText `db:"lists"` } // Subscription represents a list attached to a subscriber. type Subscription struct { List SubscriptionStatus null.String `db:"subscription_status" json:"subscription_status"` SubscriptionCreatedAt null.String `db:"subscription_created_at" json:"subscription_created_at"` Meta json.RawMessage `db:"meta" json:"meta"` } // SubscriberExportProfile represents a subscriber's collated data in JSON for export. type SubscriberExportProfile struct { Email string `db:"email" json:"-"` Profile json.RawMessage `db:"profile" json:"profile,omitempty"` Subscriptions json.RawMessage `db:"subscriptions" json:"subscriptions,omitempty"` CampaignViews json.RawMessage `db:"campaign_views" json:"campaign_views,omitempty"` LinkClicks json.RawMessage `db:"link_clicks" json:"link_clicks,omitempty"` } // JSON is the wrapper for reading and writing arbitrary JSONB fields from the DB. type JSON map[string]interface{} // StringIntMap is used to define DB Scan()s. type StringIntMap map[string]int // Subscribers represents a slice of Subscriber. type Subscribers []Subscriber // SubscriberExport represents a subscriber record that is exported to raw data. type SubscriberExport struct { Base UUID string `db:"uuid" json:"uuid"` Email string `db:"email" json:"email"` Name string `db:"name" json:"name"` Attribs string `db:"attribs" json:"attribs"` Status string `db:"status" json:"status"` } // List represents a mailing list. type List struct { Base UUID string `db:"uuid" json:"uuid"` Name string `db:"name" json:"name"` Type string `db:"type" json:"type"` Optin string `db:"optin" json:"optin"` Tags pq.StringArray `db:"tags" json:"tags"` Description string `db:"description" json:"description"` SubscriberCount int `db:"-" json:"subscriber_count"` SubscriberCounts StringIntMap `db:"subscriber_statuses" json:"subscriber_statuses"` SubscriberID int `db:"subscriber_id" json:"-"` // This is only relevant when querying the lists of a subscriber. SubscriptionStatus string `db:"subscription_status" json:"subscription_status,omitempty"` SubscriptionCreatedAt null.Time `db:"subscription_created_at" json:"subscription_created_at,omitempty"` SubscriptionUpdatedAt null.Time `db:"subscription_updated_at" json:"subscription_updated_at,omitempty"` // Pseudofield for getting the total number of subscribers // in searches and queries. Total int `db:"total" json:"-"` } // Campaign represents an e-mail campaign. type Campaign struct { Base CampaignMeta UUID string `db:"uuid" json:"uuid"` Type string `db:"type" json:"type"` Name string `db:"name" json:"name"` Subject string `db:"subject" json:"subject"` FromEmail string `db:"from_email" json:"from_email"` Body string `db:"body" json:"body"` AltBody null.String `db:"altbody" json:"altbody"` SendAt null.Time `db:"send_at" json:"send_at"` Status string `db:"status" json:"status"` ContentType string `db:"content_type" json:"content_type"` Tags pq.StringArray `db:"tags" json:"tags"` Headers Headers `db:"headers" json:"headers"` TemplateID int `db:"template_id" json:"template_id"` Messenger string `db:"messenger" json:"messenger"` Archive bool `db:"archive" json:"archive"` ArchiveSlug null.String `db:"archive_slug" json:"archive_slug"` ArchiveTemplateID int `db:"archive_template_id" json:"archive_template_id"` ArchiveMeta json.RawMessage `db:"archive_meta" json:"archive_meta"` // TemplateBody is joined in from templates by the next-campaigns query. TemplateBody string `db:"template_body" json:"-"` ArchiveTemplateBody string `db:"archive_template_body" json:"-"` Tpl *template.Template `json:"-"` SubjectTpl *txttpl.Template `json:"-"` AltBodyTpl *template.Template `json:"-"` // List of media (attachment) IDs obtained from the next-campaign query // while sending a campaign. MediaIDs pq.Int64Array `json:"-" db:"media_id"` // Fetched bodies of the attachments. Attachments []Attachment `json:"-" db:"-"` // Pseudofield for getting the total number of subscribers // in searches and queries. Total int `db:"total" json:"-"` } // CampaignMeta contains fields tracking a campaign's progress. type CampaignMeta struct { CampaignID int `db:"campaign_id" json:"-"` Views int `db:"views" json:"views"` Clicks int `db:"clicks" json:"clicks"` Bounces int `db:"bounces" json:"bounces"` // This is a list of {list_id, name} pairs unlike Subscriber.Lists[] // because lists can be deleted after a campaign is finished, resulting // in null lists data to be returned. For that reason, campaign_lists maintains // campaign-list associations with a historical record of id + name that persist // even after a list is deleted. Lists types.JSONText `db:"lists" json:"lists"` Media types.JSONText `db:"media" json:"media"` StartedAt null.Time `db:"started_at" json:"started_at"` ToSend int `db:"to_send" json:"to_send"` Sent int `db:"sent" json:"sent"` } type CampaignStats struct { ID int `db:"id" json:"id"` Status string `db:"status" json:"status"` ToSend int `db:"to_send" json:"to_send"` Sent int `db:"sent" json:"sent"` Started null.Time `db:"started_at" json:"started_at"` UpdatedAt null.Time `db:"updated_at" json:"updated_at"` Rate int `json:"rate"` NetRate int `json:"net_rate"` } type CampaignAnalyticsCount struct { CampaignID int `db:"campaign_id" json:"campaign_id"` Count int `db:"count" json:"count"` Timestamp time.Time `db:"timestamp" json:"timestamp"` } type CampaignAnalyticsLink struct { URL string `db:"url" json:"url"` Count int `db:"count" json:"count"` } // Campaigns represents a slice of Campaigns. type Campaigns []Campaign // Template represents a reusable e-mail template. type Template struct { Base Name string `db:"name" json:"name"` // Subject is only for type=tx. Subject string `db:"subject" json:"subject"` Type string `db:"type" json:"type"` Body string `db:"body" json:"body,omitempty"` IsDefault bool `db:"is_default" json:"is_default"` // Only relevant to tx (transactional) templates. SubjectTpl *txttpl.Template `json:"-"` Tpl *template.Template `json:"-"` } // Bounce represents a single bounce event. type Bounce struct { ID int `db:"id" json:"id"` Type string `db:"type" json:"type"` Source string `db:"source" json:"source"` Meta json.RawMessage `db:"meta" json:"meta"` CreatedAt time.Time `db:"created_at" json:"created_at"` // One of these should be provided. Email string `db:"email" json:"email,omitempty"` SubscriberUUID string `db:"subscriber_uuid" json:"subscriber_uuid,omitempty"` SubscriberID int `db:"subscriber_id" json:"subscriber_id,omitempty"` CampaignUUID string `db:"campaign_uuid" json:"campaign_uuid,omitempty"` Campaign *json.RawMessage `db:"campaign" json:"campaign"` // Pseudofield for getting the total number of bounces // in searches and queries. Total int `db:"total" json:"-"` } // Message is the message pushed to a Messenger. type Message struct { From string To []string Subject string ContentType string Body []byte AltBody []byte Headers textproto.MIMEHeader Attachments []Attachment Subscriber Subscriber // Campaign is generally the same instance for a large number of subscribers. Campaign *Campaign // Messenger is the messenger backend to use: email|postback. Messenger string } // Attachment represents a file or blob attachment that can be // sent along with a message by a Messenger. type Attachment struct { Name string Header textproto.MIMEHeader Content []byte } // TxMessage represents an e-mail campaign. type TxMessage struct { SubscriberEmails []string `json:"subscriber_emails"` SubscriberIDs []int `json:"subscriber_ids"` // Deprecated. SubscriberEmail string `json:"subscriber_email"` SubscriberID int `json:"subscriber_id"` TemplateID int `json:"template_id"` Data map[string]interface{} `json:"data"` FromEmail string `json:"from_email"` Headers Headers `json:"headers"` ContentType string `json:"content_type"` Messenger string `json:"messenger"` // File attachments added from multi-part form data. Attachments []Attachment `json:"-"` Subject string `json:"-"` Body []byte `json:"-"` Tpl *template.Template `json:"-"` SubjectTpl *txttpl.Template `json:"-"` } // markdown is a global instance of Markdown parser and renderer. var markdown = goldmark.New( goldmark.WithParserOptions( parser.WithAutoHeadingID(), ), goldmark.WithRendererOptions( html.WithXHTML(), html.WithUnsafe(), ), goldmark.WithExtensions( extension.Table, extension.Strikethrough, extension.TaskList, extension.NewTypographer( extension.WithTypographicSubstitutions(extension.TypographicSubstitutions{ extension.LeftDoubleQuote: []byte(`"`), extension.RightDoubleQuote: []byte(`"`), }), ), ), ) // GetIDs returns the list of subscriber IDs. func (subs Subscribers) GetIDs() []int { IDs := make([]int, len(subs)) for i, c := range subs { IDs[i] = c.ID } return IDs } // LoadLists lazy loads the lists for all the subscribers // in the Subscribers slice and attaches them to their []Lists property. func (subs Subscribers) LoadLists(stmt *sqlx.Stmt) error { var sl []subLists err := stmt.Select(&sl, pq.Array(subs.GetIDs())) if err != nil { return err } if len(subs) != len(sl) { return errors.New("campaign stats count does not match") } for i, s := range sl { if s.SubscriberID == subs[i].ID { subs[i].Lists = s.Lists } } return nil } // Value returns the JSON marshalled SubscriberAttribs. func (s JSON) Value() (driver.Value, error) { return json.Marshal(s) } // Scan unmarshals JSONB from the DB. func (s JSON) Scan(src interface{}) error { if src == nil { s = make(JSON) return nil } if data, ok := src.([]byte); ok { return json.Unmarshal(data, &s) } return fmt.Errorf("could not not decode type %T -> %T", src, s) } // Scan unmarshals JSONB from the DB. func (s StringIntMap) Scan(src interface{}) error { if src == nil { s = make(StringIntMap) return nil } if data, ok := src.([]byte); ok { return json.Unmarshal(data, &s) } return fmt.Errorf("could not not decode type %T -> %T", src, s) } // GetIDs returns the list of campaign IDs. func (camps Campaigns) GetIDs() []int { IDs := make([]int, len(camps)) for i, c := range camps { IDs[i] = c.ID } return IDs } // LoadStats lazy loads campaign stats onto a list of campaigns. func (camps Campaigns) LoadStats(stmt *sqlx.Stmt) error { var meta []CampaignMeta if err := stmt.Select(&meta, pq.Array(camps.GetIDs())); err != nil { return err } if len(camps) != len(meta) { return errors.New("campaign stats count does not match") } for i, c := range meta { if c.CampaignID == camps[i].ID { camps[i].Lists = c.Lists camps[i].Views = c.Views camps[i].Clicks = c.Clicks camps[i].Bounces = c.Bounces camps[i].Media = c.Media } } return nil } // CompileTemplate compiles a campaign body template into its base // template and sets the resultant template to Campaign.Tpl. func (c *Campaign) CompileTemplate(f template.FuncMap) error { // If the subject line has a template string, compile it. if strings.Contains(c.Subject, "{{") { subj := c.Subject for _, r := range regTplFuncs { subj = r.regExp.ReplaceAllString(subj, r.replace) } var txtFuncs map[string]interface{} = f subjTpl, err := txttpl.New(ContentTpl).Funcs(txtFuncs).Parse(subj) if err != nil { return fmt.Errorf("error compiling subject: %v", err) } c.SubjectTpl = subjTpl } // Compile the base template. body := c.TemplateBody for _, r := range regTplFuncs { body = r.regExp.ReplaceAllString(body, r.replace) } baseTPL, err := template.New(BaseTpl).Funcs(f).Parse(body) if err != nil { return fmt.Errorf("error compiling base template: %v", err) } // If the format is markdown, convert Markdown to HTML. if c.ContentType == CampaignContentTypeMarkdown { var b bytes.Buffer if err := markdown.Convert([]byte(c.Body), &b); err != nil { return err } body = b.String() } else { body = c.Body } // Compile the campaign message. for _, r := range regTplFuncs { body = r.regExp.ReplaceAllString(body, r.replace) } msgTpl, err := template.New(ContentTpl).Funcs(f).Parse(body) if err != nil { return fmt.Errorf("error compiling message: %v", err) } out, err := baseTPL.AddParseTree(ContentTpl, msgTpl.Tree) if err != nil { return fmt.Errorf("error inserting child template: %v", err) } c.Tpl = out if strings.Contains(c.AltBody.String, "{{") { b := c.AltBody.String for _, r := range regTplFuncs { b = r.regExp.ReplaceAllString(b, r.replace) } bTpl, err := template.New(ContentTpl).Funcs(f).Parse(b) if err != nil { return fmt.Errorf("error compiling alt plaintext message: %v", err) } c.AltBodyTpl = bTpl } return nil } // ConvertContent converts a campaign's body from one format to another, // for example, Markdown to HTML. func (c *Campaign) ConvertContent(from, to string) (string, error) { body := c.Body for _, r := range regTplFuncs { body = r.regExp.ReplaceAllString(body, r.replace) } // If the format is markdown, convert Markdown to HTML. var out string if from == CampaignContentTypeMarkdown && (to == CampaignContentTypeHTML || to == CampaignContentTypeRichtext) { var b bytes.Buffer if err := markdown.Convert([]byte(c.Body), &b); err != nil { return out, err } out = b.String() } else { return out, errors.New("unknown formats to convert") } return out, nil } // Compile compiles a template body and subject (only for tx templates) and // caches the templat references to be executed later. func (t *Template) Compile(f template.FuncMap) error { tpl, err := template.New(BaseTpl).Funcs(f).Parse(t.Body) if err != nil { return fmt.Errorf("error compiling transactional template: %v", err) } t.Tpl = tpl // If the subject line has a template string, compile it. if strings.Contains(t.Subject, "{{") { subj := t.Subject subjTpl, err := txttpl.New(BaseTpl).Funcs(txttpl.FuncMap(f)).Parse(subj) if err != nil { return fmt.Errorf("error compiling subject: %v", err) } t.SubjectTpl = subjTpl } return nil } func (m *TxMessage) Render(sub Subscriber, tpl *Template) error { data := struct { Subscriber Subscriber Tx *TxMessage }{sub, m} // Render the body. b := bytes.Buffer{} if err := tpl.Tpl.ExecuteTemplate(&b, BaseTpl, data); err != nil { return err } m.Body = make([]byte, b.Len()) copy(m.Body, b.Bytes()) b.Reset() // If the subject is also a template, render that. if tpl.SubjectTpl != nil { if err := tpl.SubjectTpl.ExecuteTemplate(&b, BaseTpl, data); err != nil { return err } m.Subject = b.String() b.Reset() } else { m.Subject = tpl.Subject } return nil } // FirstName splits the name by spaces and returns the first chunk // of the name that's greater than 2 characters in length, assuming // that it is the subscriber's first name. func (s Subscriber) FirstName() string { for _, s := range strings.Split(s.Name, " ") { if len(s) > 2 { return s } } return s.Name } // LastName splits the name by spaces and returns the last chunk // of the name that's greater than 2 characters in length, assuming // that it is the subscriber's last name. func (s Subscriber) LastName() string { chunks := strings.Split(s.Name, " ") for i := len(chunks) - 1; i >= 0; i-- { chunk := chunks[i] if len(chunk) > 2 { return chunk } } return s.Name } // Scan implements the sql.Scanner interface. func (h *Headers) Scan(src interface{}) error { var b []byte switch src := src.(type) { case []byte: b = src case string: b = []byte(src) case nil: return nil } if err := json.Unmarshal(b, h); err != nil { return err } return nil } // Value implements the driver.Valuer interface. func (h Headers) Value() (driver.Value, error) { if h == nil { return nil, nil } if n := len(h); n > 0 { b, err := json.Marshal(h) if err != nil { return nil, err } return b, nil } return "[]", nil }