From b033b94091c623f1e3b1dae0e5e80b4a49107020 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 30 Jun 2016 09:29:21 -0700 Subject: [PATCH] Break Category into Folder, Label, populate Gmail lables for messages --- packages/nylas-api/routes/categories.js | 14 +- packages/nylas-api/routes/messages.js | 8 +- packages/nylas-api/routes/threads.js | 28 ++-- packages/nylas-api/serialization.js | 11 +- packages/nylas-core/imap-connection.js | 8 +- packages/nylas-core/models/account/file.js | 8 +- .../models/account/{category.js => folder.js} | 13 +- packages/nylas-core/models/account/label.js | 28 ++++ packages/nylas-core/models/account/message.js | 50 +++++-- .../models/account/thread-category.js | 7 - packages/nylas-core/models/account/thread.js | 18 ++- packages/nylas-message-processor/app.js | 18 ++- .../processors/threading.js | 76 +++++++--- .../nylas-sync/imap/fetch-category-list.js | 28 ++-- .../imap/fetch-messages-in-category.js | 141 ++++++++++-------- packages/nylas-sync/sync-worker.js | 20 +-- .../mark-message-as-read.imap.js | 2 +- .../mark-message-as-unread.imap.js | 2 +- .../mark-thread-as-read.imap.js | 2 +- .../mark-thread-as-unread.imap.js | 2 +- .../move-message-to-folder.imap.js | 4 +- .../syncback_tasks/move-to-folder.imap.js | 4 +- .../syncback_tasks/star-message.imap.js | 2 +- .../syncback_tasks/star-thread.imap.js | 2 +- .../nylas-sync/syncback_tasks/task-helpers.js | 10 +- .../syncback_tasks/unstar-message.imap.js | 2 +- .../syncback_tasks/unstar-thread.imap.js | 2 +- 27 files changed, 318 insertions(+), 192 deletions(-) rename packages/nylas-core/models/account/{category.js => folder.js} (65%) create mode 100644 packages/nylas-core/models/account/label.js delete mode 100644 packages/nylas-core/models/account/thread-category.js diff --git a/packages/nylas-api/routes/categories.js b/packages/nylas-api/routes/categories.js index 65d9ccdaa..aa6374bbb 100644 --- a/packages/nylas-api/routes/categories.js +++ b/packages/nylas-api/routes/categories.js @@ -2,7 +2,9 @@ const Joi = require('joi'); const Serialization = require('../serialization'); module.exports = (server) => { - ['folders', 'labels'].forEach((term) => { + ['Folder', 'Label'].forEach((klass) => { + const term = `${klass.toLowerCase()}s`; + server.route({ method: 'GET', path: `/${term}`, @@ -18,18 +20,18 @@ module.exports = (server) => { }, response: { schema: Joi.array().items( - Serialization.jsonSchema('Category') + Serialization.jsonSchema(klass) ), }, }, handler: (request, reply) => { request.getAccountDatabase().then((db) => { - const {Category} = db; - Category.findAll({ + const Klass = db[klass]; + Klass.findAll({ limit: request.query.limit, offset: request.query.offset, - }).then((categories) => { - reply(Serialization.jsonStringify(categories)); + }).then((items) => { + reply(Serialization.jsonStringify(items)); }) }) }, diff --git a/packages/nylas-api/routes/messages.js b/packages/nylas-api/routes/messages.js index e4fb5aaf1..e5f0575e3 100644 --- a/packages/nylas-api/routes/messages.js +++ b/packages/nylas-api/routes/messages.js @@ -25,11 +25,11 @@ module.exports = (server) => { }, handler: (request, reply) => { request.getAccountDatabase().then((db) => { - const {Message, Category} = db; + const {Message, Folder, Label} = db; Message.findAll({ limit: request.query.limit, offset: request.query.offset, - include: {model: Category}, + include: [{model: Folder}, {model: Label}], }).then((messages) => { reply(Serialization.jsonStringify(messages)); }) @@ -58,12 +58,12 @@ module.exports = (server) => { }, handler: (request, reply) => { request.getAccountDatabase().then((db) => { - const {Message, Category} = db; + const {Message, Folder, Label} = db; const {headers: {accept}} = request; const {params: {id}} = request; const account = request.auth.credentials; - Message.findOne({where: {id}, include: {model: Category}}).then((message) => { + Message.findOne({where: {id}, include: [{model: Folder}, {model: Label}]}).then((message) => { if (!message) { return reply.notFound(`Message ${id} not found`) } diff --git a/packages/nylas-api/routes/threads.js b/packages/nylas-api/routes/threads.js index 1d0ae8780..0c7ad8c36 100644 --- a/packages/nylas-api/routes/threads.js +++ b/packages/nylas-api/routes/threads.js @@ -40,7 +40,7 @@ module.exports = (server) => { }, handler: (request, reply) => { request.getAccountDatabase().then((db) => { - const {Thread, Category, Message} = db; + const {Thread, Folder, Label, Message} = db; const query = request.query; const where = {}; const include = []; @@ -90,16 +90,18 @@ module.exports = (server) => { // Association queries if (query.in) { - include.push({ - model: Category, - where: { $or: [ - { id: query.in }, - { name: query.in }, - { role: query.in }, - ]}, - }); + // BEN TODO FIX BEFORE COMMITTING + // include.push({ + // model: Folder, + // where: { $or: [ + // { id: query.in }, + // { name: query.in }, + // { role: query.in }, + // ]}, + // }); } else { - include.push({model: Category}) + include.push({model: Folder}) + include.push({model: Label}) } if (query.view === 'expanded') { @@ -132,12 +134,12 @@ module.exports = (server) => { where: where, include: include, }).then((threads) => { - // if the user requested the expanded viw, fill message.category using - // thread.category, since it must be a superset. + // if the user requested the expanded viw, fill message.folder using + // thread.folders, since it must be a superset. if (query.view === 'expanded') { for (const thread of threads) { for (const msg of thread.messages) { - msg.category = thread.categories.find(c => c.id === msg.categoryId); + msg.folder = thread.folders.find(c => c.id === msg.folderId); } } } diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index a11add152..9570d6519 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -22,7 +22,16 @@ function jsonSchema(modelName) { sync_error: Joi.object(), }) } - if (modelName === 'Category') { + if (modelName === 'Folder') { + return Joi.object().keys({ + id: Joi.number(), + object: Joi.string(), + account_id: Joi.string(), + name: Joi.string().allow(null), + display_name: Joi.string(), + }) + } + if (modelName === 'Label') { return Joi.object().keys({ id: Joi.number(), object: Joi.string(), diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index 4f6fcd64f..8da026230 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -135,11 +135,11 @@ class IMAPBox { return this._imap.delFlagsAsync(range, flags) } - moveFromBox(range, categoryName) { + moveFromBox(range, folderName) { if (!this._imap) { throw new Error(`IMAPBox::moveFromBox - You need to call connect() first.`) } - return this._imap.moveAsync(range, categoryName) + return this._imap.moveAsync(range, folderName) } closeBox({expunge = true} = {}) { @@ -267,11 +267,11 @@ class IMAPConnection extends EventEmitter { /** * @return {Promise} that resolves to instance of IMAPBox */ - openBox(categoryName, {readOnly = false} = {}) { + openBox(folderName, {readOnly = false} = {}) { if (!this._imap) { throw new Error(`IMAPConnection::openBox - You need to call connect() first.`) } - return this._imap.openBoxAsync(categoryName, readOnly).then((box) => + return this._imap.openBoxAsync(folderName, readOnly).then((box) => new IMAPBox(this._imap, box) ) } diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js index ca9dcb2c8..90e196c90 100644 --- a/packages/nylas-core/models/account/file.js +++ b/packages/nylas-core/models/account/file.js @@ -23,14 +23,14 @@ module.exports = (sequelize, Sequelize) => { connection: IMAPConnection.connect(db, settings), }) .then(({message, connection}) => { - return message.getCategory() - .then((category) => connection.openBox(category.name)) + return message.getFolder() + .then((folder) => connection.openBox(folder.name)) .then((imapBox) => imapBox.fetchStream({ - messageId: message.categoryUID, + messageId: message.folderUID, options: { bodies: [this.partId], struct: true, - } + }, })) .then((stream) => { if (stream) { diff --git a/packages/nylas-core/models/account/category.js b/packages/nylas-core/models/account/folder.js similarity index 65% rename from packages/nylas-core/models/account/category.js rename to packages/nylas-core/models/account/folder.js index be7795ae1..5cec31d41 100644 --- a/packages/nylas-core/models/account/category.js +++ b/packages/nylas-core/models/account/folder.js @@ -1,18 +1,17 @@ const {JSONType} = require('../../database-types'); module.exports = (sequelize, Sequelize) => { - const Category = sequelize.define('category', { + const Folder = sequelize.define('folder', { accountId: { type: Sequelize.STRING, allowNull: false }, version: Sequelize.INTEGER, name: Sequelize.STRING, role: Sequelize.STRING, - type: Sequelize.ENUM('folder', 'label'), syncState: JSONType('syncState'), }, { classMethods: { - associate: ({Message, Thread, ThreadCategory}) => { - Category.hasMany(Message) - Category.belongsToMany(Thread, {through: ThreadCategory}) + associate: ({Message, Thread}) => { + Folder.hasMany(Message) + Folder.belongsToMany(Thread, {through: 'thread_folders'}) }, }, instanceMethods: { @@ -20,7 +19,7 @@ module.exports = (sequelize, Sequelize) => { return { id: this.id, account_id: this.accountId, - object: this.type, + object: 'folder', name: this.role, display_name: this.name, }; @@ -28,5 +27,5 @@ module.exports = (sequelize, Sequelize) => { }, }); - return Category; + return Folder; }; diff --git a/packages/nylas-core/models/account/label.js b/packages/nylas-core/models/account/label.js new file mode 100644 index 000000000..647955210 --- /dev/null +++ b/packages/nylas-core/models/account/label.js @@ -0,0 +1,28 @@ +module.exports = (sequelize, Sequelize) => { + const Label = sequelize.define('label', { + accountId: { type: Sequelize.STRING, allowNull: false }, + version: Sequelize.INTEGER, + name: Sequelize.STRING, + role: Sequelize.STRING, + }, { + classMethods: { + associate: ({Message, Thread}) => { + Label.belongsToMany(Message, {through: 'message_labels'}) + Label.belongsToMany(Thread, {through: 'thread_labels'}) + }, + }, + instanceMethods: { + toJSON: function toJSON() { + return { + id: this.id, + account_id: this.accountId, + object: 'label', + name: this.role, + display_name: this.name, + }; + }, + }, + }); + + return Label; +}; diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index 3b7791436..da092fa9a 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -23,7 +23,8 @@ module.exports = (sequelize, Sequelize) => { cc: JSONARRAYType('cc'), bcc: JSONARRAYType('bcc'), replyTo: JSONARRAYType('replyTo'), - categoryImapUID: { type: Sequelize.STRING, allowNull: true}, + folderImapUID: { type: Sequelize.STRING, allowNull: true}, + folderImapXGMLabels: { type: Sequelize.STRING, allowNull: true}, }, { indexes: [ { @@ -32,25 +33,52 @@ module.exports = (sequelize, Sequelize) => { }, ], classMethods: { - associate: ({Category, File, Thread}) => { - Message.belongsTo(Category) - Message.hasMany(File, {as: 'files'}) + associate: ({Folder, Label, File, Thread}) => { Message.belongsTo(Thread) + Message.belongsTo(Folder) + Message.belongsToMany(Label, {through: 'message_labels'}) + Message.hasMany(File) }, hashForHeaders: (headers) => { return crypto.createHash('sha256').update(headers, 'utf8').digest('hex'); }, }, instanceMethods: { + setLabelsFromXGM(xGmLabels, {preloadedLabels} = {}) { + if (!xGmLabels) { + return Promise.resolve(); + } + const labelNames = xGmLabels.filter(l => l[0] !== '\\') + const labelRoles = xGmLabels.filter(l => l[0] === '\\').map(l => l.substr(1).toLowerCase()) + const Label = sequelize.models.label; + + let getLabels = null; + if (preloadedLabels) { + getLabels = Promise.resolve(preloadedLabels.filter(l => labelNames.includes(l.name) || labelRoles.includes(l.role))); + } else { + getLabels = Label.findAll({ + where: sequelize.or({name: labelNames}, {role: labelRoles}), + }) + } + + this.folderImapXGMLabels = JSON.stringify(xGmLabels); + + return getLabels.then((labels) => + this.save().then(() => + this.setLabels(labels) + ) + ) + }, + fetchRaw: function fetchRaw({account, db}) { const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials()) return Promise.props({ - category: this.getCategory(), + folder: this.getFolder(), connection: IMAPConnection.connect(db, settings), }) - .then(({category, connection}) => { - return connection.openBox(category.name) - .then((imapBox) => imapBox.fetchMessage(this.categoryImapUID)) + .then(({folder, connection}) => { + return connection.openBox(folder.name) + .then((imapBox) => imapBox.fetchMessage(this.folderImapUID)) .then((message) => { if (message) { return Promise.resolve(`${message.headers}${message.body}`) @@ -62,8 +90,8 @@ module.exports = (sequelize, Sequelize) => { }, toJSON: function toJSON() { - if (this.category_id && !this.category) { - throw new Error("Message.toJSON called on a message where category were not eagerly loaded.") + if (this.folder_id && !this.folder) { + throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.") } return { @@ -81,7 +109,7 @@ module.exports = (sequelize, Sequelize) => { date: this.date.getTime() / 1000.0, unread: this.unread, starred: this.starred, - folder: this.category, + folder: this.folder, }; }, }, diff --git a/packages/nylas-core/models/account/thread-category.js b/packages/nylas-core/models/account/thread-category.js deleted file mode 100644 index 29d8f941a..000000000 --- a/packages/nylas-core/models/account/thread-category.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = (sequelize, Sequelize) => { - const ThreadCategory = sequelize.define('threadCategory', { - role: Sequelize.STRING, - }); - - return ThreadCategory; -}; diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js index 71f65b35f..6e9be1cca 100644 --- a/packages/nylas-core/models/account/thread.js +++ b/packages/nylas-core/models/account/thread.js @@ -20,15 +20,19 @@ module.exports = (sequelize, Sequelize) => { { fields: ['threadId'] }, ], classMethods: { - associate: ({Category, Message, ThreadCategory}) => { - Thread.belongsToMany(Category, {through: ThreadCategory}) - Thread.hasMany(Message, {as: 'messages'}) + associate: ({Folder, Label, Message}) => { + Thread.belongsToMany(Folder, {through: 'thread_folders'}) + Thread.belongsToMany(Label, {through: 'thread_labels'}) + Thread.hasMany(Message) }, }, instanceMethods: { toJSON: function toJSON() { - if (!(this.categories instanceof Array)) { - throw new Error("Thread.toJSON called on a thread where categories were not eagerly loaded.") + if (!(this.labels instanceof Array)) { + throw new Error("Thread.toJSON called on a thread where labels were not eagerly loaded.") + } + if (!(this.folders instanceof Array)) { + throw new Error("Thread.toJSON called on a thread where folders were not eagerly loaded.") } if (!(this.messages instanceof Array)) { throw new Error("Thread.toJSON called on a thread where messages were not eagerly loaded. (Only need the IDs!)") @@ -37,8 +41,8 @@ module.exports = (sequelize, Sequelize) => { const response = { id: this.id, object: 'thread', - folders: this.categories.filter(c => c.type === 'folder'), - labels: this.categories.filter(c => c.type === 'label'), + folders: this.folders, + labels: this.labels, account_id: this.accountId, participants: this.participants, subject: this.subject, diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js index 295328f23..4eab60fc8 100644 --- a/packages/nylas-message-processor/app.js +++ b/packages/nylas-message-processor/app.js @@ -11,10 +11,6 @@ const MessageAttributes = ['body', 'processed', 'to', 'from', 'cc', 'replyTo', ' const MessageProcessorVersion = 1; function runPipeline({db, accountId, message}) { - if (!message) { - return Promise.reject(new Error(`Message not found: ${message.id}`)) - } - console.log(`Processing message ${message.id}`) return processors.reduce((prevPromise, processor) => ( prevPromise.then((prevMessage) => { @@ -41,7 +37,7 @@ function saveMessage(message) { function dequeueJob() { const conn = PubsubConnector.buildClient() - conn.brpopAsync('message-processor-queue', 10000).then((item) => { + conn.brpopAsync('message-processor-queue', 10).then((item) => { if (!item) { return dequeueJob(); } @@ -56,13 +52,19 @@ function dequeueJob() { const {messageId, accountId} = json; DatabaseConnector.forAccount(accountId).then((db) => - db.Message.find({where: {id: messageId}}).then((message) => - runPipeline({db, accountId, message}).then((processedMessage) => + db.Message.find({ + where: {id: messageId}, + include: [{model: db.Folder}, {model: db.Label}], + }).then((message) => { + if (!message) { + return Promise.reject(new Error(`Message not found (${messageId}). Maybe account was deleted?`)) + } + return runPipeline({db, accountId, message}).then((processedMessage) => saveMessage(processedMessage) ).catch((err) => console.error(`MessageProcessor Failed: ${err} ${err.stack}`) ) - ) + }) ).finally(() => { dequeueJob() }); diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 6e0ee84c8..57e896ec2 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -28,8 +28,15 @@ class ThreadingProcessor { return subject.replace(regex, () => ""); } + emptyThread(Thread, options = {}) { + const t = Thread.build(options) + t.folders = []; + t.labels = []; + return t; + } + findOrCreateByMatching(db, message) { - const {Thread} = db + const {Thread, Label, Folder} = db // in the future, we should look at In-reply-to. Problem is it's a single- // directional linked list, and we don't scan the mailbox from oldest=>newest, @@ -43,19 +50,31 @@ class ThreadingProcessor { order: [ ['id', 'DESC'], ], - limit: 50, + limit: 10, + include: [{model: Label}, {model: Folder}], }).then((threads) => - this.pickMatchingThread(message, threads) || Thread.build({}) + this.pickMatchingThread(message, threads) || this.emptyThread(Thread) ) } - findOrCreateByThreadId({Thread}, threadId) { - return Thread.find({where: {threadId}}).then((thread) => { - return thread || Thread.build({threadId}); + findOrCreateByThreadId({Thread, Label, Folder}, threadId) { + return Thread.find({ + where: {threadId}, + include: [{model: Label}, {model: Folder}], + }).then((thread) => { + return thread || this.emptyThread(Thread, {threadId}) }) } processMessage({db, message}) { + if (!(message.labels instanceof Array)) { + throw new Error("Threading processMessage expects labels to be an inflated array."); + } + if (message.folder === undefined) { + throw new Error("Threading processMessage expects folder value to be present."); + } + + const {Folder, Label} = db; let findOrCreateThread = null; if (message.headers['x-gm-thrid']) { findOrCreateThread = this.findOrCreateByThreadId(db, message.headers['x-gm-thrid']) @@ -65,11 +84,19 @@ class ThreadingProcessor { return Promise.props({ thread: findOrCreateThread, - sentCategory: db.Category.find({where: {role: 'sent'}}), + sentFolder: Folder.find({where: {role: 'sent'}}), + sentLabel: Label.find({where: {role: 'sent'}}), }) - .then(({thread, sentCategory}) => { + .then(({thread, sentFolder, sentLabel}) => { thread.addMessage(message); + if (!(thread.labels instanceof Array)) { + throw new Error("Threading processMessage expects thread.labels to be an inflated array."); + } + if (!(thread.folders instanceof Array)) { + throw new Error("Threading processMessage expects thread.folders to be an inflated array."); + } + // update the basic properties of the thread thread.accountId = message.accountId; @@ -100,23 +127,34 @@ class ThreadingProcessor { if (!thread.firstMessageDate || (message.date < thread.firstMessageDate)) { thread.firstMessageDate = message.date; } - const sentCategoryId = sentCategory ? sentCategory.id : null; - if ((message.categoryId === sentCategoryId) && (message.date > thread.lastMessageSentDate)) { + + let isSent = false; + if (sentFolder) { + isSent = message.folderId === sentFolder.id + } else if (sentLabel) { + isSent = !!message.labels.find(l => l.id === sentLabel.id) + } + + if (isSent && (message.date > thread.lastMessageSentDate)) { thread.lastMessageSentDate = message.date; } - if ((message.categoryId !== sentCategoryId) && (message.date > thread.lastMessageReceivedDate)) { + if (!isSent && (message.date > thread.lastMessageReceivedDate)) { thread.lastMessageReceivedDate = message.date; } - // update categories and sav - return thread.hasCategory(message.categoryId).then((hasCategory) => { - if (!hasCategory) { - thread.addCategory(message.categoryId) + // update folders and labels + if (!thread.folders.find(f => f.id === message.folderId)) { + thread.addFolder(message.folder) + } + for (const label of message.labels) { + if (!thread.labels.find(l => l.id === label)) { + thread.addLabel(label) } - return thread.save().then((saved) => { - message.threadId = saved.id; - return message; - }); + } + + return thread.save().then((saved) => { + message.threadId = saved.id; + return message; }); }); } diff --git a/packages/nylas-sync/imap/fetch-category-list.js b/packages/nylas-sync/imap/fetch-category-list.js index 613d27308..b82f33876 100644 --- a/packages/nylas-sync/imap/fetch-category-list.js +++ b/packages/nylas-sync/imap/fetch-category-list.js @@ -2,20 +2,20 @@ const {Provider} = require('nylas-core'); const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam']; -class FetchCategoryList { +class FetchFolderList { constructor(provider) { this._provider = provider; } description() { - return `FetchCategoryList`; + return `FetchFolderList`; } - _typeForMailbox(boxName) { + _classForMailbox(boxName, box, {Folder, Label}) { if (this._provider === Provider.Gmail) { - return GMAIL_FOLDERS.includes(boxName) ? 'folder' : 'label'; + return GMAIL_FOLDERS.includes(boxName) ? Folder : Label; } - return 'folder'; + return Folder; } _roleForMailbox(boxName, box) { @@ -40,8 +40,6 @@ class FetchCategoryList { } _updateCategoriesWithBoxes(categories, boxes) { - const {Category} = this._db; - const stack = []; const created = []; const next = []; @@ -66,10 +64,10 @@ class FetchCategoryList { let category = categories.find((cat) => cat.name === boxName); if (!category) { - category = Category.build({ + const Klass = this._classForMailbox(boxName, box, this._db); + category = Klass.build({ name: boxName, accountId: this._db.accountId, - type: this._typeForMailbox(boxName, box), role: this._roleForMailbox(boxName, box), }); created.push(category); @@ -87,11 +85,15 @@ class FetchCategoryList { this._db = db; return imap.getBoxes().then((boxes) => { - const {Category, sequelize} = this._db; + const {Folder, Label, sequelize} = this._db; return sequelize.transaction((transaction) => { - return Category.findAll({transaction}).then((categories) => { - const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes); + return Promise.props({ + folders: Folder.findAll({transaction}), + labels: Label.findAll({transaction}), + }).then(({folders, labels}) => { + const all = [].concat(folders, labels); + const {created, deleted} = this._updateCategoriesWithBoxes(all, boxes); let promises = [Promise.resolve()] promises = promises.concat(created.map(cat => cat.save({transaction}))) @@ -103,4 +105,4 @@ class FetchCategoryList { } } -module.exports = FetchCategoryList; +module.exports = FetchFolderList; diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-category.js index 57737e171..ce8aa540d 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -4,9 +4,9 @@ const Imap = require('imap'); const {IMAPConnection, PubsubConnector} = require('nylas-core'); const {Capabilities} = IMAPConnection; -const MessageFlagAttributes = ['id', 'categoryImapUID', 'unread', 'starred'] +const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels'] -class FetchMessagesInCategory { +class FetchMessagesInFolder { constructor(category, options) { this._imap = null this._box = null @@ -14,12 +14,12 @@ class FetchMessagesInCategory { this._category = category; this._options = options; if (!this._category) { - throw new NylasError("FetchMessagesInCategory requires a category") + throw new NylasError("FetchMessagesInFolder requires a category") } } description() { - return `FetchMessagesInCategory (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; + return `FetchMessagesInFolder (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; } _getLowerBoundUID(count) { @@ -34,64 +34,78 @@ class FetchMessagesInCategory { const {Message} = this._db; return this._db.sequelize.transaction((transaction) => Message.update({ - categoryImapUID: null, - categoryId: null, + folderImapUID: null, + folderId: null, }, { transaction: transaction, where: { - categoryId: this._category.id, + folderId: this._category.id, }, }) ) } _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) { + const {sequelize, Label} = this._db; + const messageAttributesMap = {}; for (const msg of localMessageAttributes) { - messageAttributesMap[msg.categoryImapUID] = msg; + messageAttributesMap[msg.folderImapUID] = msg; } const createdUIDs = []; - const changedMessages = []; + const flagChangeMessages = []; - Object.keys(remoteUIDAttributes).forEach((uid) => { - const msg = messageAttributesMap[uid]; - const flags = remoteUIDAttributes[uid].flags; + return Label.findAll().then((preloadedLabels) => { + Object.keys(remoteUIDAttributes).forEach((uid) => { + const msg = messageAttributesMap[uid]; + const attrs = remoteUIDAttributes[uid]; - if (!msg) { - createdUIDs.push(uid); - return; + if (!msg) { + createdUIDs.push(uid); + return; + } + + const unread = !attrs.flags.includes('\\Seen'); + const starred = attrs.flags.includes('\\Flagged'); + const xGmLabels = attrs['x-gm-labels']; + const xGmLabelsJSON = xGmLabels ? JSON.stringify(xGmLabels) : null; + + if (msg.folderImapXGMLabels !== xGmLabelsJSON) { + msg.setLabelsFromXGM(xGmLabels, {preloadedLabels}); + } + + if (msg.unread !== unread || msg.starred !== starred) { + msg.unread = unread; + msg.starred = starred; + flagChangeMessages.push(msg); + } + }) + + console.log(` --- found ${flagChangeMessages.length || 'no'} flag changes`) + if (createdUIDs.length > 0) { + console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`) } - const unread = !flags.includes('\\Seen'); - const starred = flags.includes('\\Flagged'); - - if (msg.unread !== unread || msg.starred !== starred) { - msg.unread = unread; - msg.starred = starred; - changedMessages.push(msg); + if (flagChangeMessages.length === 0) { + return Promise.resolve(); } - }) - console.log(` --- found ${changedMessages.length || 'no'} flag changes`) - if (createdUIDs.length > 0) { - console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`) - } - - return this._db.sequelize.transaction((transaction) => - Promise.all(changedMessages.map(m => m.save({ - fields: MessageFlagAttributes, - transaction, - }))) - ); + return sequelize.transaction((transaction) => + Promise.all(flagChangeMessages.map(m => m.save({ + fields: MessageFlagAttributes, + transaction, + }))) + ); + }); } _removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) { const {Message} = this._db; const removedUIDs = localMessageAttributes - .filter(msg => !remoteUIDAttributes[msg.categoryImapUID]) - .map(msg => msg.categoryImapUID) + .filter(msg => !remoteUIDAttributes[msg.folderImapUID]) + .map(msg => msg.folderImapUID) console.log(` --- found ${removedUIDs.length} messages no longer in the folder`) @@ -100,13 +114,13 @@ class FetchMessagesInCategory { } return this._db.sequelize.transaction((transaction) => Message.update({ - categoryImapUID: null, - categoryId: null, + folderImapUID: null, + folderId: null, }, { transaction, where: { - categoryId: this._category.id, - categoryImapUID: removedUIDs, + folderId: this._category.id, + folderImapUID: removedUIDs, }, }) ); @@ -146,7 +160,6 @@ class FetchMessagesInCategory { return; } const key = JSON.stringify(desiredParts); - console.log(key); uidsByPart[key] = uidsByPart[key] || []; uidsByPart[key].push(attributes.uid); }); @@ -163,7 +176,6 @@ class FetchMessagesInCategory { const $body = this._box.fetch(uids, {bodies, struct: true}) $body.subscribe((msg) => { - console.log(`Fetched message ${msg.attributes.uid}`) msg.body = {}; for (const {id, mimetype} of desiredParts) { msg.body[mimetype] = msg.parts[id]; @@ -223,24 +235,31 @@ class FetchMessagesInCategory { unread: !attributes.flags.includes('\\Seen'), starred: attributes.flags.includes('\\Flagged'), date: attributes.date, - categoryImapUID: attributes.uid, - categoryId: this._category.id, + folderImapUID: attributes.uid, + folderId: this._category.id, headers: parsedHeaders, headerMessageId: parsedHeaders['message-id'][0], subject: parsedHeaders.subject[0], } - Message.find({where: {hash}}).then((existing) => { - if (existing) { - Object.assign(existing, values); - existing.save(); - return; - } + let created = false; - Message.create(values).then((created) => { - this._createFilesFromStruct({message: created, struct: attributes.struct}) - PubsubConnector.queueProcessMessage({accountId, messageId: created.id}); - }) + Message.find({where: {hash}}) + .then((existing) => { + created = existing != null; + return existing ? existing.update(values) : Message.create(values); + }) + .then((message) => + message.setLabelsFromXGM(attributes['x-gm-labels']).thenReturn(message) + ) + .then((message) => { + if (created) { + console.log(`Created message ID: ${message.id}, UID: ${attributes.uid}`) + this._createFilesFromStruct({message, struct: attributes.struct}) + PubsubConnector.queueProcessMessage({accountId, messageId: message.id}); + } else { + console.log(`Updated message ID: ${message.id}, UID: ${attributes.uid}`) + } }) return null; @@ -295,7 +314,7 @@ class FetchMessagesInCategory { return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => { const {fetchedmin, fetchedmax} = this._category.syncState; - return this.updateCategorySyncState({ + return this.updateFolderSyncState({ fetchedmin: fetchedmin ? Math.min(fetchedmin, min) : min, fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max, uidvalidity: boxUidvalidity, @@ -342,7 +361,7 @@ class FetchMessagesInCategory { return shallowFetch .then((remoteUIDAttributes) => ( this._db.Message.findAll({ - where: {categoryId: this._category.id}, + where: {folderId: this._category.id}, attributes: MessageFlagAttributes, }) .then((localMessageAttributes) => ( @@ -350,7 +369,7 @@ class FetchMessagesInCategory { )) .then(() => { console.log(` - finished fetching changes to messages`); - return this.updateCategorySyncState({ + return this.updateFolderSyncState({ highestmodseq: nextHighestmodseq, timeShallowScan: Date.now(), }) @@ -368,7 +387,7 @@ class FetchMessagesInCategory { return this._box.fetchUIDAttributes(range) .then((remoteUIDAttributes) => { return Message.findAll({ - where: {categoryId: this._category.id}, + where: {folderId: this._category.id}, attributes: MessageFlagAttributes, }) .then((localMessageAttributes) => ( @@ -379,7 +398,7 @@ class FetchMessagesInCategory { )) .then(() => { console.log(` - Deep scan finished.`); - return this.updateCategorySyncState({ + return this.updateFolderSyncState({ highestmodseq: this._box.highestmodseq, timeDeepScan: Date.now(), timeShallowScan: Date.now(), @@ -388,7 +407,7 @@ class FetchMessagesInCategory { }); } - updateCategorySyncState(newState) { + updateFolderSyncState(newState) { if (_.isMatch(this._category.syncState, newState)) { return Promise.resolve(); } @@ -409,4 +428,4 @@ class FetchMessagesInCategory { } } -module.exports = FetchMessagesInCategory; +module.exports = FetchMessagesInFolder; diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index ab91656be..2b96c4b04 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -6,8 +6,8 @@ const { MessageTypes, } = require('nylas-core'); -const FetchCategoryList = require('./imap/fetch-category-list') -const FetchMessagesInCategory = require('./imap/fetch-messages-in-category') +const FetchFolderList = require('./imap/fetch-category-list') +const FetchMessagesInFolder = require('./imap/fetch-messages-in-category') const SyncbackTaskFactory = require('./syncback-task-factory') @@ -69,8 +69,8 @@ class SyncWorker { const {afterSync} = this._account.syncPolicy; if (afterSync === 'idle') { - return this.getInboxCategory() - .then((inboxCategory) => this._conn.openBox(inboxCategory.name)) + return this.getIdleFolder() + .then((idleFolder) => this._conn.openBox(idleFolder.name)) .then(() => console.log('SyncWorker: - Idling on inbox category')) .catch((error) => { console.error('SyncWorker: - Unhandled error while attempting to idle on Inbox after sync: ', error) @@ -91,8 +91,8 @@ class SyncWorker { this.syncNow(); } - getInboxCategory() { - return this._db.Category.find({where: {role: 'inbox'}}) + getIdleFolder() { + return this._db.Folder.find({where: {role: ['all', 'inbox']}}) } ensureConnection() { @@ -143,23 +143,23 @@ class SyncWorker { } syncAllCategories() { - const {Category} = this._db; + const {Folder} = this._db; const {folderSyncOptions} = this._account.syncPolicy; - return Category.findAll({where: {type: 'folder'}}).then((categories) => { + return Folder.findAll().then((categories) => { const priority = ['inbox', 'all', 'drafts', 'sent', 'spam', 'trash'].reverse(); const categoriesToSync = categories.sort((a, b) => (priority.indexOf(a.role) - priority.indexOf(b.role)) * -1 ) return Promise.all(categoriesToSync.map((cat) => - this._conn.runOperation(new FetchMessagesInCategory(cat, folderSyncOptions)) + this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions)) )) }); } performSync() { - return this._conn.runOperation(new FetchCategoryList(this._account.provider)) + return this._conn.runOperation(new FetchFolderList(this._account.provider)) .then(() => this.syncbackMessageActions()) .then(() => this.syncAllCategories()) } diff --git a/packages/nylas-sync/syncback_tasks/mark-message-as-read.imap.js b/packages/nylas-sync/syncback_tasks/mark-message-as-read.imap.js index 2b513bc33..26dfb6faf 100644 --- a/packages/nylas-sync/syncback_tasks/mark-message-as-read.imap.js +++ b/packages/nylas-sync/syncback_tasks/mark-message-as-read.imap.js @@ -11,7 +11,7 @@ class MarkMessageAsReadIMAP extends SyncbackTask { return TaskHelpers.openMessageBox({messageId, db, imap}) .then(({box, message}) => { - return box.addFlags(message.categoryImapUID, 'SEEN') + return box.addFlags(message.folderImapUID, 'SEEN') }) } } diff --git a/packages/nylas-sync/syncback_tasks/mark-message-as-unread.imap.js b/packages/nylas-sync/syncback_tasks/mark-message-as-unread.imap.js index 56b6ff763..f7f7a3484 100644 --- a/packages/nylas-sync/syncback_tasks/mark-message-as-unread.imap.js +++ b/packages/nylas-sync/syncback_tasks/mark-message-as-unread.imap.js @@ -11,7 +11,7 @@ class MarkMessageAsUnreadIMAP extends SyncbackTask { return TaskHelpers.openMessageBox({messageId, db, imap}) .then(({box, message}) => { - return box.delFlags(message.categoryImapUID, 'SEEN') + return box.delFlags(message.folderImapUID, 'SEEN') }) } } diff --git a/packages/nylas-sync/syncback_tasks/mark-thread-as-read.imap.js b/packages/nylas-sync/syncback_tasks/mark-thread-as-read.imap.js index 2862e033e..c74ac249c 100644 --- a/packages/nylas-sync/syncback_tasks/mark-thread-as-read.imap.js +++ b/packages/nylas-sync/syncback_tasks/mark-thread-as-read.imap.js @@ -10,7 +10,7 @@ class MarkThreadAsRead extends SyncbackTask { const threadId = this.syncbackRequestObject().props.threadId const eachMsg = ({message, box}) => { - return box.addFlags(message.categoryImapUID, 'SEEN') + return box.addFlags(message.folderImapUID, 'SEEN') } return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg}) diff --git a/packages/nylas-sync/syncback_tasks/mark-thread-as-unread.imap.js b/packages/nylas-sync/syncback_tasks/mark-thread-as-unread.imap.js index d5748e5e4..72ae4068a 100644 --- a/packages/nylas-sync/syncback_tasks/mark-thread-as-unread.imap.js +++ b/packages/nylas-sync/syncback_tasks/mark-thread-as-unread.imap.js @@ -10,7 +10,7 @@ class MarkThreadAsUnread extends SyncbackTask { const threadId = this.syncbackRequestObject().props.threadId const eachMsg = ({message, box}) => { - return box.delFlags(message.categoryImapUID, 'SEEN') + return box.delFlags(message.folderImapUID, 'SEEN') } return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg}) diff --git a/packages/nylas-sync/syncback_tasks/move-message-to-folder.imap.js b/packages/nylas-sync/syncback_tasks/move-message-to-folder.imap.js index 8cdb412e6..4f342d047 100644 --- a/packages/nylas-sync/syncback_tasks/move-message-to-folder.imap.js +++ b/packages/nylas-sync/syncback_tasks/move-message-to-folder.imap.js @@ -12,8 +12,8 @@ class MoveMessageToFolderIMAP extends SyncbackTask { return TaskHelpers.openMessageBox({messageId, db, imap}) .then(({box, message}) => { - return db.Category.findById(toFolderId).then((newCategory) => { - return box.moveFromBox(message.categoryImapUID, newCategory.name) + return db.Folder.findById(toFolderId).then((newFolder) => { + return box.moveFromBox(message.folderImapUID, newFolder.name) }) }) } diff --git a/packages/nylas-sync/syncback_tasks/move-to-folder.imap.js b/packages/nylas-sync/syncback_tasks/move-to-folder.imap.js index 2ad4bd767..989d534c7 100644 --- a/packages/nylas-sync/syncback_tasks/move-to-folder.imap.js +++ b/packages/nylas-sync/syncback_tasks/move-to-folder.imap.js @@ -11,8 +11,8 @@ class MoveToFolderIMAP extends SyncbackTask { const toFolderId = this.syncbackRequestObject().props.folderId const eachMsg = ({message, box}) => { - return db.Category.findById(toFolderId).then((category) => { - return box.moveFromBox(message.categoryImapUID, category.name) + return db.Folder.findById(toFolderId).then((category) => { + return box.moveFromBox(message.folderImapUID, category.name) }) } diff --git a/packages/nylas-sync/syncback_tasks/star-message.imap.js b/packages/nylas-sync/syncback_tasks/star-message.imap.js index 81b2e7a31..8ae5a4f59 100644 --- a/packages/nylas-sync/syncback_tasks/star-message.imap.js +++ b/packages/nylas-sync/syncback_tasks/star-message.imap.js @@ -11,7 +11,7 @@ class StarMessageIMAP extends SyncbackTask { return TaskHelpers.openMessageBox({messageId, db, imap}) .then(({box, message}) => { - return box.addFlags(message.categoryImapUID, 'FLAGGED') + return box.addFlags(message.folderImapUID, 'FLAGGED') }) } } diff --git a/packages/nylas-sync/syncback_tasks/star-thread.imap.js b/packages/nylas-sync/syncback_tasks/star-thread.imap.js index c18becdd9..71083f96f 100644 --- a/packages/nylas-sync/syncback_tasks/star-thread.imap.js +++ b/packages/nylas-sync/syncback_tasks/star-thread.imap.js @@ -10,7 +10,7 @@ class StarThread extends SyncbackTask { const threadId = this.syncbackRequestObject().props.threadId const eachMsg = ({message, box}) => { - return box.addFlags(message.categoryImapUID, 'FLAGGED') + return box.addFlags(message.folderImapUID, 'FLAGGED') } return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg}) diff --git a/packages/nylas-sync/syncback_tasks/task-helpers.js b/packages/nylas-sync/syncback_tasks/task-helpers.js index ccff3a279..66a8759d8 100644 --- a/packages/nylas-sync/syncback_tasks/task-helpers.js +++ b/packages/nylas-sync/syncback_tasks/task-helpers.js @@ -1,19 +1,19 @@ const _ = require('underscore') const TaskHelpers = { - messagesForThreadByCategory: function messagesForThreadByCategory(db, threadId) { + messagesForThreadByFolder: function messagesForThreadByFolder(db, threadId) { return db.Thread.findById(threadId).then((thread) => { return thread.getMessages() }).then((messages) => { - return _.groupBy(messages, "categoryId") + return _.groupBy(messages, "folderId") }) }, forEachMessageInThread: function forEachMessageInThread({threadId, db, imap, callback}) { - return TaskHelpers.messagesForThreadByCategory(db, threadId) + return TaskHelpers.messagesForThreadByFolder(db, threadId) .then((msgsInCategories) => { const cids = Object.keys(msgsInCategories); - return db.Category.findAll({where: {id: cids}}) + return db.Folder.findAll({where: {id: cids}}) .each((category) => imap.openBox(category.name, {readOnly: false}).then((box) => { return Promise.all(msgsInCategories[category.id].map((message) => @@ -26,7 +26,7 @@ const TaskHelpers = { openMessageBox: function openMessageBox({messageId, db, imap}) { return db.Message.findById(messageId).then((message) => { - return db.Category.findById(message.categoryId).then((category) => { + return db.Folder.findById(message.folderId).then((category) => { return imap.openBox(category.name).then((box) => { return Promise.resolve({box, message}) }) diff --git a/packages/nylas-sync/syncback_tasks/unstar-message.imap.js b/packages/nylas-sync/syncback_tasks/unstar-message.imap.js index 3c8fb7828..c4f9cbd85 100644 --- a/packages/nylas-sync/syncback_tasks/unstar-message.imap.js +++ b/packages/nylas-sync/syncback_tasks/unstar-message.imap.js @@ -11,7 +11,7 @@ class UnstarMessageIMAP extends SyncbackTask { return TaskHelpers.openMessageBox({messageId, db, imap}) .then(({box, message}) => { - return box.delFlags(message.categoryImapUID, 'FLAGGED') + return box.delFlags(message.folderImapUID, 'FLAGGED') }) } } diff --git a/packages/nylas-sync/syncback_tasks/unstar-thread.imap.js b/packages/nylas-sync/syncback_tasks/unstar-thread.imap.js index b1d205073..c1652318c 100644 --- a/packages/nylas-sync/syncback_tasks/unstar-thread.imap.js +++ b/packages/nylas-sync/syncback_tasks/unstar-thread.imap.js @@ -10,7 +10,7 @@ class UnstarThread extends SyncbackTask { const threadId = this.syncbackRequestObject().props.threadId const eachMsg = ({message, box}) => { - return box.delFlags(message.categoryImapUID, 'FLAGGED') + return box.delFlags(message.folderImapUID, 'FLAGGED') } return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})