Foundry376/Mailspring
1
1
Fork 0
mirror of https://github.com/Foundry376/Mailspring.git synced 2025-01-01 13:14:16 +08:00
Code Issues Projects Releases Packages Wiki Activity

Break Category into Folder, Label, populate Gmail lables for messages

Browse source
This commit is contained in:
Ben Gotow 2016-06-30 09:29:21 -07:00
parent c9c52fae1b
commit b033b94091
27 changed files with 318 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
packages/nylas-api/routes/categories.js
View file

@ -2,7 +2,9 @@ const Joi = require('joi');
const Serialization = require('../serialization');
module.exports = (server) => {
['folders', 'labels'].forEach((term) => {
['Folder', 'Label'].forEach((klass) => {
const term = `${klass.toLowerCase()}s`;
server.route({
method: 'GET',
path: `/${term}`,
@ -18,18 +20,18 @@ module.exports = (server) => {
},
response: {
schema: Joi.array().items(
Serialization.jsonSchema('Category')
Serialization.jsonSchema(klass)
),
},
},
handler: (request, reply) => {
request.getAccountDatabase().then((db) => {
const {Category} = db;
Category.findAll({
const Klass = db[klass];
Klass.findAll({
limit: request.query.limit,
offset: request.query.offset,
}).then((categories) => {
reply(Serialization.jsonStringify(categories));
}).then((items) => {
reply(Serialization.jsonStringify(items));
})
})
},

8
packages/nylas-api/routes/messages.js
View file

@ -25,11 +25,11 @@ module.exports = (server) => {
},
handler: (request, reply) => {
request.getAccountDatabase().then((db) => {
const {Message, Category} = db;
const {Message, Folder, Label} = db;
Message.findAll({
limit: request.query.limit,
offset: request.query.offset,
include: {model: Category},
include: [{model: Folder}, {model: Label}],
}).then((messages) => {
reply(Serialization.jsonStringify(messages));
})
@ -58,12 +58,12 @@ module.exports = (server) => {
},
handler: (request, reply) => {
request.getAccountDatabase().then((db) => {
const {Message, Category} = db;
const {Message, Folder, Label} = db;
const {headers: {accept}} = request;
const {params: {id}} = request;
const account = request.auth.credentials;
Message.findOne({where: {id}, include: {model: Category}}).then((message) => {
Message.findOne({where: {id}, include: [{model: Folder}, {model: Label}]}).then((message) => {
if (!message) {
return reply.notFound(`Message ${id} not found`)
}

28
packages/nylas-api/routes/threads.js
View file

@ -40,7 +40,7 @@ module.exports = (server) => {
},
handler: (request, reply) => {
request.getAccountDatabase().then((db) => {
const {Thread, Category, Message} = db;
const {Thread, Folder, Label, Message} = db;
const query = request.query;
const where = {};
const include = [];
@ -90,16 +90,18 @@ module.exports = (server) => {
// Association queries
if (query.in) {
include.push({
model: Category,
where: { $or: [
{ id: query.in },
{ name: query.in },
{ role: query.in },
]},
});
// BEN TODO FIX BEFORE COMMITTING
// include.push({
// model: Folder,
// where: { $or: [
// { id: query.in },
// { name: query.in },
// { role: query.in },
// ]},
// });
} else {
include.push({model: Category})
include.push({model: Folder})
include.push({model: Label})
}
if (query.view === 'expanded') {
@ -132,12 +134,12 @@ module.exports = (server) => {
where: where,
include: include,
}).then((threads) => {
// if the user requested the expanded viw, fill message.category using
// thread.category, since it must be a superset.
// if the user requested the expanded viw, fill message.folder using
// thread.folders, since it must be a superset.
if (query.view === 'expanded') {
for (const thread of threads) {
for (const msg of thread.messages) {
msg.category = thread.categories.find(c => c.id === msg.categoryId);
msg.folder = thread.folders.find(c => c.id === msg.folderId);
}
}
}

11
packages/nylas-api/serialization.js
View file

@ -22,7 +22,16 @@ function jsonSchema(modelName) {
sync_error: Joi.object(),
})
}
if (modelName === 'Category') {
if (modelName === 'Folder') {
return Joi.object().keys({
id: Joi.number(),
object: Joi.string(),
account_id: Joi.string(),
name: Joi.string().allow(null),
display_name: Joi.string(),
})
}
if (modelName === 'Label') {
return Joi.object().keys({
id: Joi.number(),
object: Joi.string(),

8
packages/nylas-core/imap-connection.js
View file

@ -135,11 +135,11 @@ class IMAPBox {
return this._imap.delFlagsAsync(range, flags)
}
moveFromBox(range, categoryName) {
moveFromBox(range, folderName) {
if (!this._imap) {
throw new Error(`IMAPBox::moveFromBox - You need to call connect() first.`)
}
return this._imap.moveAsync(range, categoryName)
return this._imap.moveAsync(range, folderName)
}
closeBox({expunge = true} = {}) {
@ -267,11 +267,11 @@ class IMAPConnection extends EventEmitter {
/**
* @return {Promise} that resolves to instance of IMAPBox
*/
openBox(categoryName, {readOnly = false} = {}) {
openBox(folderName, {readOnly = false} = {}) {
if (!this._imap) {
throw new Error(`IMAPConnection::openBox - You need to call connect() first.`)
}
return this._imap.openBoxAsync(categoryName, readOnly).then((box) =>
return this._imap.openBoxAsync(folderName, readOnly).then((box) =>
new IMAPBox(this._imap, box)
)
}

8
packages/nylas-core/models/account/file.js
View file

@ -23,14 +23,14 @@ module.exports = (sequelize, Sequelize) => {
connection: IMAPConnection.connect(db, settings),
})
.then(({message, connection}) => {
return message.getCategory()
.then((category) => connection.openBox(category.name))
return message.getFolder()
.then((folder) => connection.openBox(folder.name))
.then((imapBox) => imapBox.fetchStream({
messageId: message.categoryUID,
messageId: message.folderUID,
options: {
bodies: [this.partId],
struct: true,
}
},
}))
.then((stream) => {
if (stream) {

13
packages/nylas-core/models/account/category.js → packages/nylas-core/models/account/folder.js
View file

@ -1,18 +1,17 @@
const {JSONType} = require('../../database-types');
module.exports = (sequelize, Sequelize) => {
const Category = sequelize.define('category', {
const Folder = sequelize.define('folder', {
accountId: { type: Sequelize.STRING, allowNull: false },
version: Sequelize.INTEGER,
name: Sequelize.STRING,
role: Sequelize.STRING,
type: Sequelize.ENUM('folder', 'label'),
syncState: JSONType('syncState'),
}, {
classMethods: {
associate: ({Message, Thread, ThreadCategory}) => {
Category.hasMany(Message)
Category.belongsToMany(Thread, {through: ThreadCategory})
associate: ({Message, Thread}) => {
Folder.hasMany(Message)
Folder.belongsToMany(Thread, {through: 'thread_folders'})
},
},
instanceMethods: {
@ -20,7 +19,7 @@ module.exports = (sequelize, Sequelize) => {
return {
id: this.id,
account_id: this.accountId,
object: this.type,
object: 'folder',
name: this.role,
display_name: this.name,
};
@ -28,5 +27,5 @@ module.exports = (sequelize, Sequelize) => {
},
});
return Category;
return Folder;
};

28
packages/nylas-core/models/account/label.js Normal file
View file

@ -0,0 +1,28 @@
module.exports = (sequelize, Sequelize) => {
const Label = sequelize.define('label', {
accountId: { type: Sequelize.STRING, allowNull: false },
version: Sequelize.INTEGER,
name: Sequelize.STRING,
role: Sequelize.STRING,
}, {
classMethods: {
associate: ({Message, Thread}) => {
Label.belongsToMany(Message, {through: 'message_labels'})
Label.belongsToMany(Thread, {through: 'thread_labels'})
},
},
instanceMethods: {
toJSON: function toJSON() {
return {
id: this.id,
account_id: this.accountId,
object: 'label',
name: this.role,
display_name: this.name,
};
},
},
});
return Label;
};

50
packages/nylas-core/models/account/message.js
View file

@ -23,7 +23,8 @@ module.exports = (sequelize, Sequelize) => {
cc: JSONARRAYType('cc'),
bcc: JSONARRAYType('bcc'),
replyTo: JSONARRAYType('replyTo'),
categoryImapUID: { type: Sequelize.STRING, allowNull: true},
folderImapUID: { type: Sequelize.STRING, allowNull: true},
folderImapXGMLabels: { type: Sequelize.STRING, allowNull: true},
}, {
indexes: [
{
@ -32,25 +33,52 @@ module.exports = (sequelize, Sequelize) => {
},
],
classMethods: {
associate: ({Category, File, Thread}) => {
Message.belongsTo(Category)
Message.hasMany(File, {as: 'files'})
associate: ({Folder, Label, File, Thread}) => {
Message.belongsTo(Thread)
Message.belongsTo(Folder)
Message.belongsToMany(Label, {through: 'message_labels'})
Message.hasMany(File)
},
hashForHeaders: (headers) => {
return crypto.createHash('sha256').update(headers, 'utf8').digest('hex');
},
},
instanceMethods: {
setLabelsFromXGM(xGmLabels, {preloadedLabels} = {}) {
if (!xGmLabels) {
return Promise.resolve();
}
const labelNames = xGmLabels.filter(l => l[0] !== '\\')
const labelRoles = xGmLabels.filter(l => l[0] === '\\').map(l => l.substr(1).toLowerCase())
const Label = sequelize.models.label;
let getLabels = null;
if (preloadedLabels) {
getLabels = Promise.resolve(preloadedLabels.filter(l => labelNames.includes(l.name) || labelRoles.includes(l.role)));
} else {
getLabels = Label.findAll({
where: sequelize.or({name: labelNames}, {role: labelRoles}),
})
}
this.folderImapXGMLabels = JSON.stringify(xGmLabels);
return getLabels.then((labels) =>
this.save().then(() =>
this.setLabels(labels)
)
)
},
fetchRaw: function fetchRaw({account, db}) {
const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
return Promise.props({
category: this.getCategory(),
folder: this.getFolder(),
connection: IMAPConnection.connect(db, settings),
})
.then(({category, connection}) => {
return connection.openBox(category.name)
.then((imapBox) => imapBox.fetchMessage(this.categoryImapUID))
.then(({folder, connection}) => {
return connection.openBox(folder.name)
.then((imapBox) => imapBox.fetchMessage(this.folderImapUID))
.then((message) => {
if (message) {
return Promise.resolve(`${message.headers}${message.body}`)
@ -62,8 +90,8 @@ module.exports = (sequelize, Sequelize) => {
},
toJSON: function toJSON() {
if (this.category_id && !this.category) {
throw new Error("Message.toJSON called on a message where category were not eagerly loaded.")
if (this.folder_id && !this.folder) {
throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.")
}
return {
@ -81,7 +109,7 @@ module.exports = (sequelize, Sequelize) => {
date: this.date.getTime() / 1000.0,
unread: this.unread,
starred: this.starred,
folder: this.category,
folder: this.folder,
};
},
},

7
packages/nylas-core/models/account/thread-category.js
View file

@ -1,7 +0,0 @@
module.exports = (sequelize, Sequelize) => {
const ThreadCategory = sequelize.define('threadCategory', {
role: Sequelize.STRING,
});
return ThreadCategory;
};

18
packages/nylas-core/models/account/thread.js
View file

@ -20,15 +20,19 @@ module.exports = (sequelize, Sequelize) => {
{ fields: ['threadId'] },
],
classMethods: {
associate: ({Category, Message, ThreadCategory}) => {
Thread.belongsToMany(Category, {through: ThreadCategory})
Thread.hasMany(Message, {as: 'messages'})
associate: ({Folder, Label, Message}) => {
Thread.belongsToMany(Folder, {through: 'thread_folders'})
Thread.belongsToMany(Label, {through: 'thread_labels'})
Thread.hasMany(Message)
},
},
instanceMethods: {
toJSON: function toJSON() {
if (!(this.categories instanceof Array)) {
throw new Error("Thread.toJSON called on a thread where categories were not eagerly loaded.")
if (!(this.labels instanceof Array)) {
throw new Error("Thread.toJSON called on a thread where labels were not eagerly loaded.")
}
if (!(this.folders instanceof Array)) {
throw new Error("Thread.toJSON called on a thread where folders were not eagerly loaded.")
}
if (!(this.messages instanceof Array)) {
throw new Error("Thread.toJSON called on a thread where messages were not eagerly loaded. (Only need the IDs!)")
@ -37,8 +41,8 @@ module.exports = (sequelize, Sequelize) => {
const response = {
id: this.id,
object: 'thread',
folders: this.categories.filter(c => c.type === 'folder'),
labels: this.categories.filter(c => c.type === 'label'),
folders: this.folders,
labels: this.labels,
account_id: this.accountId,
participants: this.participants,
subject: this.subject,

18
packages/nylas-message-processor/app.js
View file

@ -11,10 +11,6 @@ const MessageAttributes = ['body', 'processed', 'to', 'from', 'cc', 'replyTo', '
const MessageProcessorVersion = 1;
function runPipeline({db, accountId, message}) {
if (!message) {
return Promise.reject(new Error(`Message not found: ${message.id}`))
}
console.log(`Processing message ${message.id}`)
return processors.reduce((prevPromise, processor) => (
prevPromise.then((prevMessage) => {
@ -41,7 +37,7 @@ function saveMessage(message) {
function dequeueJob() {
const conn = PubsubConnector.buildClient()
conn.brpopAsync('message-processor-queue', 10000).then((item) => {
conn.brpopAsync('message-processor-queue', 10).then((item) => {
if (!item) {
return dequeueJob();
}
@ -56,13 +52,19 @@ function dequeueJob() {
const {messageId, accountId} = json;
DatabaseConnector.forAccount(accountId).then((db) =>
db.Message.find({where: {id: messageId}}).then((message) =>
runPipeline({db, accountId, message}).then((processedMessage) =>
db.Message.find({
where: {id: messageId},
include: [{model: db.Folder}, {model: db.Label}],
}).then((message) => {
if (!message) {
return Promise.reject(new Error(`Message not found (${messageId}). Maybe account was deleted?`))
}
return runPipeline({db, accountId, message}).then((processedMessage) =>
saveMessage(processedMessage)
).catch((err) =>
console.error(`MessageProcessor Failed: ${err} ${err.stack}`)
)
)
})
).finally(() => {
dequeueJob()
});

76
packages/nylas-message-processor/processors/threading.js
View file

@ -28,8 +28,15 @@ class ThreadingProcessor {
return subject.replace(regex, () => "");
}
emptyThread(Thread, options = {}) {
const t = Thread.build(options)
t.folders = [];
t.labels = [];
return t;
}
findOrCreateByMatching(db, message) {
const {Thread} = db
const {Thread, Label, Folder} = db
// in the future, we should look at In-reply-to. Problem is it's a single-
// directional linked list, and we don't scan the mailbox from oldest=>newest,
@ -43,19 +50,31 @@ class ThreadingProcessor {
order: [
['id', 'DESC'],
],
limit: 50,
limit: 10,
include: [{model: Label}, {model: Folder}],
}).then((threads) =>
this.pickMatchingThread(message, threads) || Thread.build({})
this.pickMatchingThread(message, threads) || this.emptyThread(Thread)
)
}
findOrCreateByThreadId({Thread}, threadId) {
return Thread.find({where: {threadId}}).then((thread) => {
return thread || Thread.build({threadId});
findOrCreateByThreadId({Thread, Label, Folder}, threadId) {
return Thread.find({
where: {threadId},
include: [{model: Label}, {model: Folder}],
}).then((thread) => {
return thread || this.emptyThread(Thread, {threadId})
})
}
processMessage({db, message}) {
if (!(message.labels instanceof Array)) {
throw new Error("Threading processMessage expects labels to be an inflated array.");
}
if (message.folder === undefined) {
throw new Error("Threading processMessage expects folder value to be present.");
}
const {Folder, Label} = db;
let findOrCreateThread = null;
if (message.headers['x-gm-thrid']) {
findOrCreateThread = this.findOrCreateByThreadId(db, message.headers['x-gm-thrid'])
@ -65,11 +84,19 @@ class ThreadingProcessor {
return Promise.props({
thread: findOrCreateThread,
sentCategory: db.Category.find({where: {role: 'sent'}}),
sentFolder: Folder.find({where: {role: 'sent'}}),
sentLabel: Label.find({where: {role: 'sent'}}),
})
.then(({thread, sentCategory}) => {
.then(({thread, sentFolder, sentLabel}) => {
thread.addMessage(message);
if (!(thread.labels instanceof Array)) {
throw new Error("Threading processMessage expects thread.labels to be an inflated array.");
}
if (!(thread.folders instanceof Array)) {
throw new Error("Threading processMessage expects thread.folders to be an inflated array.");
}
// update the basic properties of the thread
thread.accountId = message.accountId;
@ -100,23 +127,34 @@ class ThreadingProcessor {
if (!thread.firstMessageDate || (message.date < thread.firstMessageDate)) {
thread.firstMessageDate = message.date;
}
const sentCategoryId = sentCategory ? sentCategory.id : null;
if ((message.categoryId === sentCategoryId) && (message.date > thread.lastMessageSentDate)) {
let isSent = false;
if (sentFolder) {
isSent = message.folderId === sentFolder.id
} else if (sentLabel) {
isSent = !!message.labels.find(l => l.id === sentLabel.id)
}
if (isSent && (message.date > thread.lastMessageSentDate)) {
thread.lastMessageSentDate = message.date;
}
if ((message.categoryId !== sentCategoryId) && (message.date > thread.lastMessageReceivedDate)) {
if (!isSent && (message.date > thread.lastMessageReceivedDate)) {
thread.lastMessageReceivedDate = message.date;
}
// update categories and sav
return thread.hasCategory(message.categoryId).then((hasCategory) => {
if (!hasCategory) {
thread.addCategory(message.categoryId)
// update folders and labels
if (!thread.folders.find(f => f.id === message.folderId)) {
thread.addFolder(message.folder)
}
for (const label of message.labels) {
if (!thread.labels.find(l => l.id === label)) {
thread.addLabel(label)
}
return thread.save().then((saved) => {
message.threadId = saved.id;
return message;
});
}
return thread.save().then((saved) => {
message.threadId = saved.id;
return message;
});
});
}

28
packages/nylas-sync/imap/fetch-category-list.js
View file

@ -2,20 +2,20 @@ const {Provider} = require('nylas-core');
const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam'];
class FetchCategoryList {
class FetchFolderList {
constructor(provider) {
this._provider = provider;
}
description() {
return `FetchCategoryList`;
return `FetchFolderList`;
}
_typeForMailbox(boxName) {
_classForMailbox(boxName, box, {Folder, Label}) {
if (this._provider === Provider.Gmail) {
return GMAIL_FOLDERS.includes(boxName) ? 'folder' : 'label';
return GMAIL_FOLDERS.includes(boxName) ? Folder : Label;
}
return 'folder';
return Folder;
}
_roleForMailbox(boxName, box) {
@ -40,8 +40,6 @@ class FetchCategoryList {
}
_updateCategoriesWithBoxes(categories, boxes) {
const {Category} = this._db;
const stack = [];
const created = [];
const next = [];
@ -66,10 +64,10 @@ class FetchCategoryList {
let category = categories.find((cat) => cat.name === boxName);
if (!category) {
category = Category.build({
const Klass = this._classForMailbox(boxName, box, this._db);
category = Klass.build({
name: boxName,
accountId: this._db.accountId,
type: this._typeForMailbox(boxName, box),
role: this._roleForMailbox(boxName, box),
});
created.push(category);
@ -87,11 +85,15 @@ class FetchCategoryList {
this._db = db;
return imap.getBoxes().then((boxes) => {
const {Category, sequelize} = this._db;
const {Folder, Label, sequelize} = this._db;
return sequelize.transaction((transaction) => {
return Category.findAll({transaction}).then((categories) => {
const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes);
return Promise.props({
folders: Folder.findAll({transaction}),
labels: Label.findAll({transaction}),
}).then(({folders, labels}) => {
const all = [].concat(folders, labels);
const {created, deleted} = this._updateCategoriesWithBoxes(all, boxes);
let promises = [Promise.resolve()]
promises = promises.concat(created.map(cat => cat.save({transaction})))
@ -103,4 +105,4 @@ class FetchCategoryList {
}
}
module.exports = FetchCategoryList;
module.exports = FetchFolderList;

141
packages/nylas-sync/imap/fetch-messages-in-category.js
View file

@ -4,9 +4,9 @@ const Imap = require('imap');
const {IMAPConnection, PubsubConnector} = require('nylas-core');
const {Capabilities} = IMAPConnection;
const MessageFlagAttributes = ['id', 'categoryImapUID', 'unread', 'starred']
const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
class FetchMessagesInCategory {
class FetchMessagesInFolder {
constructor(category, options) {
this._imap = null
this._box = null
@ -14,12 +14,12 @@ class FetchMessagesInCategory {
this._category = category;
this._options = options;
if (!this._category) {
throw new NylasError("FetchMessagesInCategory requires a category")
throw new NylasError("FetchMessagesInFolder requires a category")
}
}
description() {
return `FetchMessagesInCategory (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`;
return `FetchMessagesInFolder (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`;
}
_getLowerBoundUID(count) {
@ -34,64 +34,78 @@ class FetchMessagesInCategory {
const {Message} = this._db;
return this._db.sequelize.transaction((transaction) =>
Message.update({
categoryImapUID: null,
categoryId: null,
folderImapUID: null,
folderId: null,
}, {
transaction: transaction,
where: {
categoryId: this._category.id,
folderId: this._category.id,
},
})
)
}
_updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
const {sequelize, Label} = this._db;
const messageAttributesMap = {};
for (const msg of localMessageAttributes) {
messageAttributesMap[msg.categoryImapUID] = msg;
messageAttributesMap[msg.folderImapUID] = msg;
}
const createdUIDs = [];
const changedMessages = [];
const flagChangeMessages = [];
Object.keys(remoteUIDAttributes).forEach((uid) => {
const msg = messageAttributesMap[uid];
const flags = remoteUIDAttributes[uid].flags;
return Label.findAll().then((preloadedLabels) => {
Object.keys(remoteUIDAttributes).forEach((uid) => {
const msg = messageAttributesMap[uid];
const attrs = remoteUIDAttributes[uid];
if (!msg) {
createdUIDs.push(uid);
return;
if (!msg) {
createdUIDs.push(uid);
return;
}
const unread = !attrs.flags.includes('\\Seen');
const starred = attrs.flags.includes('\\Flagged');
const xGmLabels = attrs['x-gm-labels'];
const xGmLabelsJSON = xGmLabels ? JSON.stringify(xGmLabels) : null;
if (msg.folderImapXGMLabels !== xGmLabelsJSON) {
msg.setLabelsFromXGM(xGmLabels, {preloadedLabels});
}
if (msg.unread !== unread || msg.starred !== starred) {
msg.unread = unread;
msg.starred = starred;
flagChangeMessages.push(msg);
}
})
console.log(` --- found ${flagChangeMessages.length || 'no'} flag changes`)
if (createdUIDs.length > 0) {
console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
}
const unread = !flags.includes('\\Seen');
const starred = flags.includes('\\Flagged');
if (msg.unread !== unread || msg.starred !== starred) {
msg.unread = unread;
msg.starred = starred;
changedMessages.push(msg);
if (flagChangeMessages.length === 0) {
return Promise.resolve();
}
})
console.log(` --- found ${changedMessages.length || 'no'} flag changes`)
if (createdUIDs.length > 0) {
console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
}
return this._db.sequelize.transaction((transaction) =>
Promise.all(changedMessages.map(m => m.save({
fields: MessageFlagAttributes,
transaction,
})))
);
return sequelize.transaction((transaction) =>
Promise.all(flagChangeMessages.map(m => m.save({
fields: MessageFlagAttributes,
transaction,
})))
);
});
}
_removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) {
const {Message} = this._db;
const removedUIDs = localMessageAttributes
.filter(msg => !remoteUIDAttributes[msg.categoryImapUID])
.map(msg => msg.categoryImapUID)
.filter(msg => !remoteUIDAttributes[msg.folderImapUID])
.map(msg => msg.folderImapUID)
console.log(` --- found ${removedUIDs.length} messages no longer in the folder`)
@ -100,13 +114,13 @@ class FetchMessagesInCategory {
}
return this._db.sequelize.transaction((transaction) =>
Message.update({
categoryImapUID: null,
categoryId: null,
folderImapUID: null,
folderId: null,
}, {
transaction,
where: {
categoryId: this._category.id,
categoryImapUID: removedUIDs,
folderId: this._category.id,
folderImapUID: removedUIDs,
},
})
);
@ -146,7 +160,6 @@ class FetchMessagesInCategory {
return;
}
const key = JSON.stringify(desiredParts);
console.log(key);
uidsByPart[key] = uidsByPart[key] || [];
uidsByPart[key].push(attributes.uid);
});
@ -163,7 +176,6 @@ class FetchMessagesInCategory {
const $body = this._box.fetch(uids, {bodies, struct: true})
$body.subscribe((msg) => {
console.log(`Fetched message ${msg.attributes.uid}`)
msg.body = {};
for (const {id, mimetype} of desiredParts) {
msg.body[mimetype] = msg.parts[id];
@ -223,24 +235,31 @@ class FetchMessagesInCategory {
unread: !attributes.flags.includes('\\Seen'),
starred: attributes.flags.includes('\\Flagged'),
date: attributes.date,
categoryImapUID: attributes.uid,
categoryId: this._category.id,
folderImapUID: attributes.uid,
folderId: this._category.id,
headers: parsedHeaders,
headerMessageId: parsedHeaders['message-id'][0],
subject: parsedHeaders.subject[0],
}
Message.find({where: {hash}}).then((existing) => {
if (existing) {
Object.assign(existing, values);
existing.save();
return;
}
let created = false;
Message.create(values).then((created) => {
this._createFilesFromStruct({message: created, struct: attributes.struct})
PubsubConnector.queueProcessMessage({accountId, messageId: created.id});
})
Message.find({where: {hash}})
.then((existing) => {
created = existing != null;
return existing ? existing.update(values) : Message.create(values);
})
.then((message) =>
message.setLabelsFromXGM(attributes['x-gm-labels']).thenReturn(message)
)
.then((message) => {
if (created) {
console.log(`Created message ID: ${message.id}, UID: ${attributes.uid}`)
this._createFilesFromStruct({message, struct: attributes.struct})
PubsubConnector.queueProcessMessage({accountId, messageId: message.id});
} else {
console.log(`Updated message ID: ${message.id}, UID: ${attributes.uid}`)
}
})
return null;
@ -295,7 +314,7 @@ class FetchMessagesInCategory {
return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => {
const {fetchedmin, fetchedmax} = this._category.syncState;
return this.updateCategorySyncState({
return this.updateFolderSyncState({
fetchedmin: fetchedmin ? Math.min(fetchedmin, min) : min,
fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max,
uidvalidity: boxUidvalidity,
@ -342,7 +361,7 @@ class FetchMessagesInCategory {
return shallowFetch
.then((remoteUIDAttributes) => (
this._db.Message.findAll({
where: {categoryId: this._category.id},
where: {folderId: this._category.id},
attributes: MessageFlagAttributes,
})
.then((localMessageAttributes) => (
@ -350,7 +369,7 @@ class FetchMessagesInCategory {
))
.then(() => {
console.log(` - finished fetching changes to messages`);
return this.updateCategorySyncState({
return this.updateFolderSyncState({
highestmodseq: nextHighestmodseq,
timeShallowScan: Date.now(),
})
@ -368,7 +387,7 @@ class FetchMessagesInCategory {
return this._box.fetchUIDAttributes(range)
.then((remoteUIDAttributes) => {
return Message.findAll({
where: {categoryId: this._category.id},
where: {folderId: this._category.id},
attributes: MessageFlagAttributes,
})
.then((localMessageAttributes) => (
@ -379,7 +398,7 @@ class FetchMessagesInCategory {
))
.then(() => {
console.log(` - Deep scan finished.`);
return this.updateCategorySyncState({
return this.updateFolderSyncState({
highestmodseq: this._box.highestmodseq,
timeDeepScan: Date.now(),
timeShallowScan: Date.now(),
@ -388,7 +407,7 @@ class FetchMessagesInCategory {
});
}
updateCategorySyncState(newState) {
updateFolderSyncState(newState) {
if (_.isMatch(this._category.syncState, newState)) {
return Promise.resolve();
}
@ -409,4 +428,4 @@ class FetchMessagesInCategory {
}
}
module.exports = FetchMessagesInCategory;
module.exports = FetchMessagesInFolder;

20
packages/nylas-sync/sync-worker.js
View file

@ -6,8 +6,8 @@ const {
MessageTypes,
} = require('nylas-core');
const FetchCategoryList = require('./imap/fetch-category-list')
const FetchMessagesInCategory = require('./imap/fetch-messages-in-category')
const FetchFolderList = require('./imap/fetch-category-list')
const FetchMessagesInFolder = require('./imap/fetch-messages-in-category')
const SyncbackTaskFactory = require('./syncback-task-factory')
@ -69,8 +69,8 @@ class SyncWorker {
const {afterSync} = this._account.syncPolicy;
if (afterSync === 'idle') {
return this.getInboxCategory()
.then((inboxCategory) => this._conn.openBox(inboxCategory.name))
return this.getIdleFolder()
.then((idleFolder) => this._conn.openBox(idleFolder.name))
.then(() => console.log('SyncWorker: - Idling on inbox category'))
.catch((error) => {
console.error('SyncWorker: - Unhandled error while attempting to idle on Inbox after sync: ', error)
@ -91,8 +91,8 @@ class SyncWorker {
this.syncNow();
}
getInboxCategory() {
return this._db.Category.find({where: {role: 'inbox'}})
getIdleFolder() {
return this._db.Folder.find({where: {role: ['all', 'inbox']}})
}
ensureConnection() {
@ -143,23 +143,23 @@ class SyncWorker {
}
syncAllCategories() {
const {Category} = this._db;
const {Folder} = this._db;
const {folderSyncOptions} = this._account.syncPolicy;
return Category.findAll({where: {type: 'folder'}}).then((categories) => {
return Folder.findAll().then((categories) => {
const priority = ['inbox', 'all', 'drafts', 'sent', 'spam', 'trash'].reverse();
const categoriesToSync = categories.sort((a, b) =>
(priority.indexOf(a.role) - priority.indexOf(b.role)) * -1
)
return Promise.all(categoriesToSync.map((cat) =>
this._conn.runOperation(new FetchMessagesInCategory(cat, folderSyncOptions))
this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions))
))
});
}
performSync() {
return this._conn.runOperation(new FetchCategoryList(this._account.provider))
return this._conn.runOperation(new FetchFolderList(this._account.provider))
.then(() => this.syncbackMessageActions())
.then(() => this.syncAllCategories())
}

2
packages/nylas-sync/syncback_tasks/mark-message-as-read.imap.js
View file

@ -11,7 +11,7 @@ class MarkMessageAsReadIMAP extends SyncbackTask {
return TaskHelpers.openMessageBox({messageId, db, imap})
.then(({box, message}) => {
return box.addFlags(message.categoryImapUID, 'SEEN')
return box.addFlags(message.folderImapUID, 'SEEN')
})
}
}

2
packages/nylas-sync/syncback_tasks/mark-message-as-unread.imap.js
View file

@ -11,7 +11,7 @@ class MarkMessageAsUnreadIMAP extends SyncbackTask {
return TaskHelpers.openMessageBox({messageId, db, imap})
.then(({box, message}) => {
return box.delFlags(message.categoryImapUID, 'SEEN')
return box.delFlags(message.folderImapUID, 'SEEN')
})
}
}

2
packages/nylas-sync/syncback_tasks/mark-thread-as-read.imap.js
View file

@ -10,7 +10,7 @@ class MarkThreadAsRead extends SyncbackTask {
const threadId = this.syncbackRequestObject().props.threadId
const eachMsg = ({message, box}) => {
return box.addFlags(message.categoryImapUID, 'SEEN')
return box.addFlags(message.folderImapUID, 'SEEN')
}
return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})

2
packages/nylas-sync/syncback_tasks/mark-thread-as-unread.imap.js
View file

@ -10,7 +10,7 @@ class MarkThreadAsUnread extends SyncbackTask {
const threadId = this.syncbackRequestObject().props.threadId
const eachMsg = ({message, box}) => {
return box.delFlags(message.categoryImapUID, 'SEEN')
return box.delFlags(message.folderImapUID, 'SEEN')
}
return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})

4
packages/nylas-sync/syncback_tasks/move-message-to-folder.imap.js
View file

@ -12,8 +12,8 @@ class MoveMessageToFolderIMAP extends SyncbackTask {
return TaskHelpers.openMessageBox({messageId, db, imap})
.then(({box, message}) => {
return db.Category.findById(toFolderId).then((newCategory) => {
return box.moveFromBox(message.categoryImapUID, newCategory.name)
return db.Folder.findById(toFolderId).then((newFolder) => {
return box.moveFromBox(message.folderImapUID, newFolder.name)
})
})
}

4
packages/nylas-sync/syncback_tasks/move-to-folder.imap.js
View file

@ -11,8 +11,8 @@ class MoveToFolderIMAP extends SyncbackTask {
const toFolderId = this.syncbackRequestObject().props.folderId
const eachMsg = ({message, box}) => {
return db.Category.findById(toFolderId).then((category) => {
return box.moveFromBox(message.categoryImapUID, category.name)
return db.Folder.findById(toFolderId).then((category) => {
return box.moveFromBox(message.folderImapUID, category.name)
})
}

2
packages/nylas-sync/syncback_tasks/star-message.imap.js
View file

@ -11,7 +11,7 @@ class StarMessageIMAP extends SyncbackTask {
return TaskHelpers.openMessageBox({messageId, db, imap})
.then(({box, message}) => {
return box.addFlags(message.categoryImapUID, 'FLAGGED')
return box.addFlags(message.folderImapUID, 'FLAGGED')
})
}
}

2
packages/nylas-sync/syncback_tasks/star-thread.imap.js
View file

@ -10,7 +10,7 @@ class StarThread extends SyncbackTask {
const threadId = this.syncbackRequestObject().props.threadId
const eachMsg = ({message, box}) => {
return box.addFlags(message.categoryImapUID, 'FLAGGED')
return box.addFlags(message.folderImapUID, 'FLAGGED')
}
return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})

10
packages/nylas-sync/syncback_tasks/task-helpers.js
View file

@ -1,19 +1,19 @@
const _ = require('underscore')
const TaskHelpers = {
messagesForThreadByCategory: function messagesForThreadByCategory(db, threadId) {
messagesForThreadByFolder: function messagesForThreadByFolder(db, threadId) {
return db.Thread.findById(threadId).then((thread) => {
return thread.getMessages()
}).then((messages) => {
return _.groupBy(messages, "categoryId")
return _.groupBy(messages, "folderId")
})
},
forEachMessageInThread: function forEachMessageInThread({threadId, db, imap, callback}) {
return TaskHelpers.messagesForThreadByCategory(db, threadId)
return TaskHelpers.messagesForThreadByFolder(db, threadId)
.then((msgsInCategories) => {
const cids = Object.keys(msgsInCategories);
return db.Category.findAll({where: {id: cids}})
return db.Folder.findAll({where: {id: cids}})
.each((category) =>
imap.openBox(category.name, {readOnly: false}).then((box) => {
return Promise.all(msgsInCategories[category.id].map((message) =>
@ -26,7 +26,7 @@ const TaskHelpers = {
openMessageBox: function openMessageBox({messageId, db, imap}) {
return db.Message.findById(messageId).then((message) => {
return db.Category.findById(message.categoryId).then((category) => {
return db.Folder.findById(message.folderId).then((category) => {
return imap.openBox(category.name).then((box) => {
return Promise.resolve({box, message})
})

2
packages/nylas-sync/syncback_tasks/unstar-message.imap.js
View file

@ -11,7 +11,7 @@ class UnstarMessageIMAP extends SyncbackTask {
return TaskHelpers.openMessageBox({messageId, db, imap})
.then(({box, message}) => {
return box.delFlags(message.categoryImapUID, 'FLAGGED')
return box.delFlags(message.folderImapUID, 'FLAGGED')
})
}
}

2
packages/nylas-sync/syncback_tasks/unstar-thread.imap.js
View file

@ -10,7 +10,7 @@ class UnstarThread extends SyncbackTask {
const threadId = this.syncbackRequestObject().props.threadId
const eachMsg = ({message, box}) => {
return box.delFlags(message.categoryImapUID, 'FLAGGED')
return box.delFlags(message.folderImapUID, 'FLAGGED')
}
return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})