-
-
Sync Policy
-
-
this.applyToAllAccounts.call(this, this.props.accountIds)}>
- Apply To All Displayed Accounts
-
-
this.cancel.call(this)}> Cancel
+
+
this.edit.call(this)}>
+ Set sync policies for currently displayed accounts
+
+
+
+
Sync Policy
+
+
this.applyToAllAccounts.call(this, this.props.accountIds)}>
+ Apply To All Displayed Accounts
+
+
this.cancel.call(this)}> Cancel
+
)
diff --git a/packages/nylas-dashboard/public/js/sync-policy.jsx b/packages/nylas-dashboard/public/js/sync-policy.jsx
index bfbee8ffb..5efe9655b 100644
--- a/packages/nylas-dashboard/public/js/sync-policy.jsx
+++ b/packages/nylas-dashboard/public/js/sync-policy.jsx
@@ -43,7 +43,7 @@ class SyncPolicy extends React.Component {
{this.props.stringifiedSyncPolicy}
this.save.call(this)}> Save
-
this.cancel.call(this)}> Cancel
+
this.cancel.call(this)}> Cancel
)
diff --git a/packages/nylas-dashboard/public/js/syncback-request-details.jsx b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
new file mode 100644
index 000000000..1aecdb220
--- /dev/null
+++ b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
@@ -0,0 +1,171 @@
+const React = window.React;
+const Dropdown = window.Dropdown;
+
+class SyncbackRequestDetails extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ open: false,
+ accountId: props.accountId,
+ syncbackRequests: null,
+ counts: null,
+ statusFilter: 'all',
+ };
+ }
+
+ getDetails() {
+ const req = new XMLHttpRequest();
+ const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}`;
+ req.open("GET", url, true);
+ req.onreadystatechange = () => {
+ if (req.readyState === XMLHttpRequest.DONE) {
+ if (req.status === 200) {
+ this.setState({syncbackRequests: req.responseText});
+ } else {
+ console.error(req.responseText);
+ }
+ }
+ }
+ req.send();
+ }
+
+ getCounts() {
+ const since = Date.now() - 1000 * 60 * 60; // one hour ago
+ const req = new XMLHttpRequest();
+ const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}/counts?since=${since}`;
+ req.open("GET", url, true);
+ req.onreadystatechange = () => {
+ if (req.readyState === XMLHttpRequest.DONE) {
+ if (req.status === 200) {
+ this.setState({counts: JSON.parse(req.responseText)});
+ } else {
+ console.error(req.responseText);
+ }
+ }
+ }
+ req.send();
+ }
+
+ setStatusFilter(statusFilter) {
+ this.setState({statusFilter: statusFilter});
+ }
+
+ open() {
+ this.getDetails();
+ this.getCounts();
+ this.setState({open: true});
+ }
+
+ close() {
+ this.setState({open: false});
+ }
+
+ render() {
+ if (this.state.open) {
+ let counts =
Of requests created in the last hour: ...
+ if (this.state.counts) {
+ const total = this.state.counts.new + this.state.counts.failed
+ + this.state.counts.succeeded;
+ if (total === 0) {
+ counts = "No requests made in the last hour";
+ } else {
+ counts = (
+
+ Of requests created in the last hour:
+
+ {this.state.counts.failed / total * 100}% failed
+
+
+ {this.state.counts.succeeded / total * 100}% succeeded
+
+
+ {/* .new was throwing off my syntax higlighting, so ignoring linter*/}
+ {this.state.counts['new'] / total * 100}% are still new
+
+
+ )
+ }
+ }
+
+ let details = "Loading..."
+ if (this.state.syncbackRequests) {
+ let reqs = JSON.parse(this.state.syncbackRequests);
+ if (this.state.statusFilter !== 'all') {
+ reqs = reqs.filter((req) => req.status === this.state.statusFilter);
+ }
+ let rows = [];
+ if (reqs.length === 0) {
+ rows.push(
No results - - );
+ }
+ for (let i = reqs.length - 1; i >= 0; i--) {
+ const req = reqs[i];
+ const date = new Date(req.createdAt);
+ rows.push(
+ {req.status}
+ {req.type}
+ {date.toLocaleTimeString()}, {date.toLocaleDateString()}
+ )
+ }
+ details = (
+
+
+
+
+ Status:
+ this.setStatusFilter.call(this, status)}
+ />
+
+ Type
+ Created At
+
+ {rows}
+
+
+ );
+ }
+
+ return (
+
+
Syncback Request Details
+
+
+
this.close.call(this)}>
+ X
+
+
+ {counts}
+ {details}
+
+
+
+
+ );
+ }
+ // else, the modal isn't open
+ return (
+
+ this.open.call(this)}>
+ Syncback Request Details
+
+
+ );
+ }
+}
+
+SyncbackRequestDetails.propTypes = {
+ accountId: React.PropTypes.number,
+}
+
+window.SyncbackRequestDetails = SyncbackRequestDetails;
diff --git a/packages/nylas-dashboard/routes/syncback-requests.js b/packages/nylas-dashboard/routes/syncback-requests.js
new file mode 100644
index 000000000..267f1086a
--- /dev/null
+++ b/packages/nylas-dashboard/routes/syncback-requests.js
@@ -0,0 +1,81 @@
+const Joi = require('joi');
+const {DatabaseConnector} = require(`nylas-core`);
+
+module.exports = (server) => {
+ server.route({
+ method: 'GET',
+ path: '/syncback-requests/{account_id}',
+ config: {
+ description: 'Get the SyncbackRequests for an account',
+ notes: 'Notes go here',
+ tags: ['syncback-requests'],
+ validate: {
+ params: {
+ account_id: Joi.number().integer(),
+ },
+ },
+ response: {
+ schema: Joi.string(),
+ },
+ },
+ handler: (request, reply) => {
+ DatabaseConnector.forAccount(request.params.account_id).then((db) => {
+ const {SyncbackRequest} = db;
+ SyncbackRequest.findAll().then((syncbackRequests) => {
+ reply(JSON.stringify(syncbackRequests))
+ });
+ });
+ },
+ });
+
+ server.route({
+ method: 'GET',
+ path: '/syncback-requests/{account_id}/counts',
+ config: {
+ description: 'Get stats on the statuses of SyncbackRequests',
+ notes: 'Notes go here',
+ tags: ['syncback-requests'],
+ validate: {
+ params: {
+ account_id: Joi.number().integer(),
+ },
+ query: {
+ since: Joi.date().timestamp(),
+ },
+ },
+ response: {
+ schema: Joi.string(),
+ },
+ },
+ handler: (request, reply) => {
+ DatabaseConnector.forAccount(request.params.account_id).then((db) => {
+ const {SyncbackRequest} = db;
+
+ const counts = {
+ 'new': null,
+ 'succeeded': null,
+ 'failed': null,
+ }
+
+ const where = {};
+ if (request.query.since) {
+ where.createdAt = {gt: request.query.since};
+ }
+
+ const countPromises = [];
+ for (const status of Object.keys(counts)) {
+ where.status = status.toUpperCase();
+ countPromises.push(
+ SyncbackRequest.count({where: where}).then((count) => {
+ counts[status] = count;
+ })
+ );
+ }
+
+ Promise.all(countPromises).then(() => {
+ reply(JSON.stringify(counts));
+ })
+ });
+ },
+ });
+};
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js
index 458a074e3..4ad7aee14 100644
--- a/packages/nylas-message-processor/app.js
+++ b/packages/nylas-message-processor/app.js
@@ -1,7 +1,8 @@
-const {PubsubConnector, DatabaseConnector} = require(`nylas-core`)
+const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
const {processors} = require('./processors')
global.Promise = require('bluebird');
+global.Logger = Logger.createLogger('nylas-k2-message-processor')
// List of the attributes of Message that the processor should be allowed to change.
// The message may move between folders, get starred, etc. while it's being
@@ -11,15 +12,13 @@ const MessageProcessorVersion = 1;
const redis = PubsubConnector.buildClient();
-function runPipeline({db, accountId, message}) {
- console.log(`Processing message ${message.id}`)
+function runPipeline({db, accountId, message, logger}) {
+ logger.info(`MessageProcessor: Processing message`)
return processors.reduce((prevPromise, processor) => (
prevPromise.then((prevMessage) => {
- const processed = processor({message: prevMessage, accountId, db});
- if (!(processed instanceof Promise)) {
- throw new Error(`processor ${processor} did not return a promise.`)
- }
- return processed.then((nextMessage) => {
+ const processed = processor({message: prevMessage, accountId, db, logger});
+ return Promise.resolve(processed)
+ .then((nextMessage) => {
if (!nextMessage.body) {
throw new Error("processor did not resolve with a valid message object.")
}
@@ -46,26 +45,28 @@ function dequeueJob() {
try {
json = JSON.parse(item[1]);
} catch (error) {
- console.error(`MessageProcessor Failed: Found invalid JSON item in queue: ${item}`)
+ global.Logger.error({item}, `MessageProcessor: Found invalid JSON item in queue`)
return dequeueJob();
}
const {messageId, accountId} = json;
+ const logger = global.Logger.forAccount({id: accountId}).child({message_id: messageId})
- DatabaseConnector.forAccount(accountId).then((db) =>
- db.Message.find({
+ DatabaseConnector.forAccount(accountId).then((db) => {
+ return db.Message.find({
where: {id: messageId},
include: [{model: db.Folder}, {model: db.Label}],
}).then((message) => {
if (!message) {
return Promise.reject(new Error(`Message not found (${messageId}). Maybe account was deleted?`))
}
- return runPipeline({db, accountId, message}).then((processedMessage) =>
+ return runPipeline({db, accountId, message, logger}).then((processedMessage) =>
saveMessage(processedMessage)
).catch((err) =>
- console.error(`MessageProcessor Failed: ${err} ${err.stack}`)
+ logger.error(err, `MessageProcessor: Failed`)
)
})
- ).finally(() => {
+ })
+ .finally(() => {
dequeueJob()
});
diff --git a/packages/nylas-message-processor/processors/parsing.js b/packages/nylas-message-processor/processors/parsing.js
index 6763437e0..a144199ab 100644
--- a/packages/nylas-message-processor/processors/parsing.js
+++ b/packages/nylas-message-processor/processors/parsing.js
@@ -11,7 +11,7 @@ function Contact({name, address} = {}) {
const extractContacts = (values) =>
(values || []).map(v => Contact(mimelib.parseAddresses(v).pop()))
-function processMessage({message}) {
+function processMessage({message, logger}) {
if (message.snippet) {
// trim and clean snippet which is alreay present (from message plaintext)
message.snippet = message.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
@@ -24,7 +24,7 @@ function processMessage({message}) {
// TODO: Fanciness
message.snippet = message.body.substr(0, Math.min(message.body.length, SNIPPET_SIZE));
} else {
- console.log("Received message has no body or snippet.")
+ logger.info("MessageProcessor: Parsing - Received message has no body or snippet.")
}
message.to = extractContacts(message.headers.to);
diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js
index 57e896ec2..2f9c847fd 100644
--- a/packages/nylas-message-processor/processors/threading.js
+++ b/packages/nylas-message-processor/processors/threading.js
@@ -9,11 +9,14 @@ class ThreadingProcessor {
// conversation. Put it back soonish.
// const messageEmails = _.uniq([].concat(message.to, message.cc, message.from).map(p => p.email));
- // console.log(`Found ${threads.length} candidate threads for message with subject: ${message.subject}`)
+ // this.logger.info({
+ // num_candidate_threads: threads.length,
+ // message_subject: message.subject,
+ // }, `Found candidate threads for message`)
//
// for (const thread of threads) {
// const threadEmails = _.uniq([].concat(thread.participants).map(p => p.email));
- // console.log(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`)
+ // this.logger.info(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`)
//
// if (_.intersection(threadEmails, messageEmails) >= threadEmails.length * 0.9) {
// return thread;
@@ -66,7 +69,7 @@ class ThreadingProcessor {
})
}
- processMessage({db, message}) {
+ processMessage({db, message, logger}) {
if (!(message.labels instanceof Array)) {
throw new Error("Threading processMessage expects labels to be an inflated array.");
}
@@ -74,6 +77,8 @@ class ThreadingProcessor {
throw new Error("Threading processMessage expects folder value to be present.");
}
+ this.logger = logger
+
const {Folder, Label} = db;
let findOrCreateThread = null;
if (message.headers['x-gm-thrid']) {
diff --git a/packages/nylas-message-processor/spec/threading-spec.js b/packages/nylas-message-processor/spec/threading-spec.js
index b1df5d815..5e0aaa3ba 100644
--- a/packages/nylas-message-processor/spec/threading-spec.js
+++ b/packages/nylas-message-processor/spec/threading-spec.js
@@ -43,7 +43,6 @@ it('adds the message to the thread', (done) => {
},
create: (message) => {
message.setThread = (thread) => {
- console.log("setting")
message.thread = thread.id
}
return Promise.resolve(message)
diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js
index 5d7319245..ac81a02f1 100644
--- a/packages/nylas-sync/app.js
+++ b/packages/nylas-sync/app.js
@@ -1,16 +1,18 @@
+// require('newrelic');
global.Promise = require('bluebird');
-
-const {DatabaseConnector} = require(`nylas-core`)
+const {DatabaseConnector, Logger} = require(`nylas-core`)
const SyncProcessManager = require('./sync-process-manager');
+global.Logger = Logger.createLogger('nylas-k2-sync')
+
const manager = new SyncProcessManager();
DatabaseConnector.forShared().then((db) => {
const {Account} = db;
Account.findAll().then((accounts) => {
if (accounts.length === 0) {
- console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`)
- console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`)
+ global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`)
+ global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`)
}
manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => {
manager.start();
diff --git a/packages/nylas-sync/imap/fetch-category-list.js b/packages/nylas-sync/imap/fetch-folder-list.js
similarity index 97%
rename from packages/nylas-sync/imap/fetch-category-list.js
rename to packages/nylas-sync/imap/fetch-folder-list.js
index b82f33876..d5c5a0000 100644
--- a/packages/nylas-sync/imap/fetch-category-list.js
+++ b/packages/nylas-sync/imap/fetch-folder-list.js
@@ -3,8 +3,9 @@ const {Provider} = require('nylas-core');
const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam'];
class FetchFolderList {
- constructor(provider) {
+ constructor(provider, logger = console) {
this._provider = provider;
+ this._logger = logger;
}
description() {
diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
similarity index 84%
rename from packages/nylas-sync/imap/fetch-messages-in-category.js
rename to packages/nylas-sync/imap/fetch-messages-in-folder.js
index 554f09e71..038dde66d 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-category.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -11,12 +11,13 @@ const FETCH_MESSAGES_FIRST_COUNT = 100;
const FETCH_MESSAGES_COUNT = 200;
class FetchMessagesInFolder {
- constructor(category, options) {
+ constructor(category, options, logger = console) {
this._imap = null
this._box = null
this._db = null
this._category = category;
this._options = options;
+ this._logger = logger;
if (!this._category) {
throw new Error("FetchMessagesInFolder requires a category")
}
@@ -86,9 +87,13 @@ class FetchMessagesInFolder {
}
})
- console.log(` --- found ${flagChangeMessages.length || 'no'} flag changes`)
+ this._logger.info({
+ flag_changes: flagChangeMessages.length,
+ }, `FetchMessagesInFolder: found flag changes`)
if (createdUIDs.length > 0) {
- console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
+ this._logger.info({
+ new_messages: createdUIDs.length,
+ }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
}
if (flagChangeMessages.length === 0) {
@@ -111,7 +116,9 @@ class FetchMessagesInFolder {
.filter(msg => !remoteUIDAttributes[msg.folderImapUID])
.map(msg => msg.folderImapUID)
- console.log(` --- found ${removedUIDs.length} messages no longer in the folder`)
+ this._logger.info({
+ removed_messages: removedUIDs.length,
+ }, `FetchMessagesInFolder: found messages no longer in the folder`)
if (removedUIDs.length === 0) {
return Promise.resolve();
@@ -148,7 +155,9 @@ class FetchMessagesInFolder {
}
if (desired.length === 0) {
- console.warn(`Could not find good part. Options are: ${available.join(', ')}`)
+ this._logger.warn({
+ available_options: available.join(', '),
+ }, `FetchMessagesInFolder: Could not find good part`)
}
return desired;
@@ -173,7 +182,10 @@ class FetchMessagesInFolder {
const uids = uidsByPart[key];
const desiredParts = JSON.parse(key);
const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
- console.log(`Fetching parts ${key} for ${uids.length} messages`)
+ this._logger.info({
+ key,
+ num_messages: uids.length,
+ }, `FetchMessagesInFolder: Fetching parts for messages`)
// note: the order of UIDs in the array doesn't matter, Gmail always
// returns them in ascending (oldest => newest) order.
@@ -258,11 +270,17 @@ class FetchMessagesInFolder {
)
.then((message) => {
if (created) {
- console.log(`Created message ID: ${message.id}, UID: ${attributes.uid}`)
+ this._logger.info({
+ message_id: message.id,
+ uid: attributes.uid,
+ }, `FetchMessagesInFolder: Created message`)
this._createFilesFromStruct({message, struct: attributes.struct})
PubsubConnector.queueProcessMessage({accountId, messageId: message.id});
} else {
- console.log(`Updated message ID: ${message.id}, UID: ${attributes.uid}`)
+ this._logger.info({
+ message_id: message.id,
+ uid: attributes.uid,
+ }, `FetchMessagesInFolder: Updated message`)
}
})
@@ -291,7 +309,9 @@ class FetchMessagesInFolder {
const desiredRanges = [];
- console.log(` - Fetching messages. Currently have range: ${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`)
+ this._logger.info({
+ range: `${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`,
+ }, `FetchMessagesInFolder: Fetching messages.`)
// Todo: In the future, this is where logic should go that limits
// sync based on number of messages / age of messages.
@@ -303,18 +323,20 @@ class FetchMessagesInFolder {
if (savedSyncState.fetchedmax < boxUidnext) {
desiredRanges.push({min: savedSyncState.fetchedmax, max: boxUidnext})
} else {
- console.log(" --- fetchedmax == uidnext, nothing more recent to fetch.")
+ this._logger.info('FetchMessagesInFolder: fetchedmax == uidnext, nothing more recent to fetch.')
}
if (savedSyncState.fetchedmin > 1) {
const lowerbound = Math.max(1, savedSyncState.fetchedmin - FETCH_MESSAGES_COUNT);
desiredRanges.push({min: lowerbound, max: savedSyncState.fetchedmin})
} else {
- console.log(" --- fetchedmin == 1, nothing older to fetch.")
+ this._logger.info("FetchMessagesInFolder: fetchedmin == 1, nothing older to fetch.")
}
}
return Promise.each(desiredRanges, ({min, max}) => {
- console.log(` --- fetching range: ${min}:${max}`);
+ this._logger.info({
+ range: `${min}:${max}`,
+ }, `FetchMessagesInFolder: Fetching range`);
return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => {
const {fetchedmin, fetchedmax} = this._category.syncState;
@@ -326,7 +348,7 @@ class FetchMessagesInFolder {
});
})
}).then(() => {
- console.log(` - Fetching messages finished`);
+ this._logger.info(`FetchMessagesInFolder: Fetching messages finished`);
});
}
@@ -350,15 +372,15 @@ class FetchMessagesInFolder {
let shallowFetch = null;
if (this._imap.serverSupports(Capabilities.Condstore)) {
- console.log(` - Shallow attribute scan (using CONDSTORE)`)
+ this._logger.info(`FetchMessagesInFolder: Shallow attribute scan (using CONDSTORE)`)
if (nextHighestmodseq === highestmodseq) {
- console.log(" --- highestmodseq matches, nothing more to fetch")
+ this._logger.info('FetchMessagesInFolder: highestmodseq matches, nothing more to fetch')
return Promise.resolve();
}
shallowFetch = this._box.fetchUIDAttributes(`1:*`, {changedsince: highestmodseq});
} else {
const range = `${this._getLowerBoundUID(SHALLOW_SCAN_UID_COUNT)}:*`;
- console.log(` - Shallow attribute scan (using range: ${range})`)
+ this._logger.info({range}, `FetchMessagesInFolder: Shallow attribute scan`)
shallowFetch = this._box.fetchUIDAttributes(range);
}
@@ -372,7 +394,7 @@ class FetchMessagesInFolder {
this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes)
))
.then(() => {
- console.log(` - finished fetching changes to messages`);
+ this._logger.info(`FetchMessagesInFolder: finished fetching changes to messages`);
return this.updateFolderSyncState({
highestmodseq: nextHighestmodseq,
timeShallowScan: Date.now(),
@@ -386,7 +408,7 @@ class FetchMessagesInFolder {
const {fetchedmin, fetchedmax} = this._category.syncState;
const range = `${fetchedmin}:${fetchedmax}`;
- console.log(` - Deep attribute scan: fetching attributes in range: ${range}`)
+ this._logger.info({range}, `FetchMessagesInFolder: Deep attribute scan: fetching attributes in range`)
return this._box.fetchUIDAttributes(range)
.then((remoteUIDAttributes) => {
@@ -401,7 +423,7 @@ class FetchMessagesInFolder {
})
))
.then(() => {
- console.log(` - Deep scan finished.`);
+ this._logger.info(`FetchMessagesInFolder: Deep scan finished.`);
return this.updateFolderSyncState({
highestmodseq: this._box.highestmodseq,
timeDeepScan: Date.now(),
diff --git a/packages/nylas-sync/newrelic.js b/packages/nylas-sync/newrelic.js
new file mode 100644
index 000000000..a85120e87
--- /dev/null
+++ b/packages/nylas-sync/newrelic.js
@@ -0,0 +1,24 @@
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+ /**
+ * Array of application names.
+ */
+ app_name: ['Nylas K2 Sync'],
+ /**
+ * Your New Relic license key.
+ */
+ license_key: 'e232d6ccc786bd87aa72b86782439710162e3739',
+ logging: {
+ /**
+ * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+ * issues with the agent, 'info' and higher will impose the least overhead on
+ * production applications.
+ */
+ level: 'info',
+ },
+}
diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index 513bd1664..8d4042895 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -40,10 +40,11 @@ class SyncProcessManager {
this._workers = {};
this._listenForSyncsClient = null;
this._exiting = false;
+ this._logger = global.Logger.child({identity: IDENTITY})
}
start() {
- console.log(`ProcessManager: Starting with ID ${IDENTITY}`)
+ this._logger.info(`ProcessManager: Starting with ID`)
this.unassignAccountsAssignedTo(IDENTITY).then(() => {
this.unassignAccountsMissingHeartbeats();
@@ -63,12 +64,14 @@ class SyncProcessManager {
client.setAsync(key, Date.now()).then(() =>
client.expireAsync(key, HEARTBEAT_EXPIRES)
).then(() =>
- console.log("ProcessManager: 💘")
+ this._logger.info({
+ accounts_syncing_count: Object.keys(this._workers).length,
+ }, "ProcessManager: 💘")
)
}
onSigInt() {
- console.log(`ProcessManager: Exiting...`)
+ this._logger.info(`ProcessManager: Exiting...`)
this._exiting = true;
this.unassignAccountsAssignedTo(IDENTITY).then(() =>
@@ -85,7 +88,7 @@ class SyncProcessManager {
let unseenIds = [].concat(accountIds);
- console.log("ProcessManager: Starting scan for accountIds in database that are not present in Redis.")
+ this._logger.info("ProcessManager: Starting scan for accountIds in database that are not present in Redis.")
return forEachAccountList((foundProcessIdentity, foundIds) => {
unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`))
@@ -94,7 +97,10 @@ class SyncProcessManager {
if (unseenIds.length === 0) {
return;
}
- console.log(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`)
+ this._logger.info({
+ unseen_ids: unseenIds.join(', '),
+ channel: ACCOUNTS_UNCLAIMED,
+ }, `ProcessManager: Adding unseen account IDs to ACCOUNTS_UNCLAIMED channel.`)
unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id));
});
}
@@ -102,7 +108,7 @@ class SyncProcessManager {
unassignAccountsMissingHeartbeats() {
const client = PubsubConnector.broadcastClient();
- console.log("ProcessManager: Starting unassignment for processes missing heartbeats.")
+ this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.")
Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, '');
@@ -125,12 +131,15 @@ class SyncProcessManager {
)
return unassignOne(0).then((returned) => {
- console.log(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`)
+ this._logger.info({
+ returned,
+ assigned_to: identity,
+ }, `ProcessManager: Returned accounts`)
});
}
update() {
- console.log(`ProcessManager: Searching for an unclaimed account to sync.`)
+ this._logger.info(`ProcessManager: Searching for an unclaimed account to sync.`)
this.acceptUnclaimedAccount().finally(() => {
if (this._exiting) {
@@ -170,7 +179,7 @@ class SyncProcessManager {
if (this._exiting || this._workers[account.id]) {
return;
}
- console.log(`ProcessManager: Starting worker for Account ${accountId}`)
+ this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`)
this._workers[account.id] = new SyncWorker(account, db, () => {
this.removeWorkerForAccountId(accountId)
});
@@ -187,7 +196,8 @@ class SyncProcessManager {
if (didRemove) {
PubsubConnector.broadcastClient().rpushAsync(dst, accountId)
} else {
- throw new Error("Wanted to return item to pool, but didn't have claim on it.")
+ this._logger.error("Wanted to return item to pool, but didn't have claim on it.")
+ return
}
this._workers[accountId] = null;
});
diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 6ebb34b8d..7178e59cf 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -11,8 +11,8 @@ const {
const {CLAIM_DURATION} = SchedulerUtils;
-const FetchFolderList = require('./imap/fetch-category-list')
-const FetchMessagesInFolder = require('./imap/fetch-messages-in-category')
+const FetchFolderList = require('./imap/fetch-folder-list')
+const FetchMessagesInFolder = require('./imap/fetch-messages-in-folder')
const SyncbackTaskFactory = require('./syncback-task-factory')
@@ -24,6 +24,7 @@ class SyncWorker {
this._startTime = Date.now();
this._lastSyncTime = null;
this._onExpired = onExpired;
+ this._logger = global.Logger.forAccount(account)
this._syncTimer = null;
this._expirationTimer = null;
@@ -54,8 +55,11 @@ class SyncWorker {
this._onAccountUpdated(); break;
case MessageTypes.SYNCBACK_REQUESTED:
this.syncNow({reason: 'Syncback Action Queued'}); break;
+ case MessageTypes.ACCOUNT_CREATED:
+ // No other processing currently required for account creation
+ break;
default:
- throw new Error(`Invalid message: ${msg}`)
+ this._logger.error({message: msg}, 'SyncWorker: Invalid message')
}
}
@@ -63,10 +67,14 @@ class SyncWorker {
if (!this.isWaitingForNextSync()) {
return;
}
- this._getAccount().then((account) => {
+ this._getAccount()
+ .then((account) => {
this._account = account;
this.syncNow({reason: 'Account Modification'});
- });
+ })
+ .catch((err) => {
+ this._logger.error(err, 'SyncWorker: Error getting account for update')
+ })
}
_onConnectionIdleUpdate() {
@@ -100,7 +108,12 @@ class SyncWorker {
return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account."))
}
- const conn = new IMAPConnection(this._db, Object.assign({}, settings, credentials));
+ const conn = new IMAPConnection({
+ db: this._db,
+ settings: Object.assign({}, settings, credentials),
+ logger: this._logger,
+ });
+
conn.on('mail', () => {
this._onConnectionIdleUpdate();
})
@@ -145,7 +158,7 @@ class SyncWorker {
)
return Promise.all(categoriesToSync.map((cat) =>
- this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions))
+ this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger))
))
});
}
@@ -155,10 +168,10 @@ class SyncWorker {
this._syncTimer = null;
if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) {
- console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) is in error state - Skipping sync`)
+ this._logger.info(`SyncWorker: Account is in error state - Skipping sync`)
return
}
- console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) sync started (${reason})`)
+ this._logger.info({reason}, `SyncWorker: Account sync started`)
this.ensureConnection()
.then(() => this._account.update({syncError: null}))
@@ -174,7 +187,7 @@ class SyncWorker {
}
onSyncError(error) {
- console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} (${this._account.id})`, error)
+ this._logger.error(error, `SyncWorker: Error while syncing account`)
this.closeConnection()
if (error.source.includes('socket') || error.source.includes('timeout')) {
@@ -189,8 +202,8 @@ class SyncWorker {
onSyncDidComplete() {
const {afterSync} = this._account.syncPolicy;
- if (!this._account.firstSyncCompletedAt) {
- this._account.firstSyncCompletedAt = Date.now()
+ if (!this._account.firstSyncCompletion) {
+ this._account.firstSyncCompletion = Date.now()
}
const now = Date.now();
@@ -203,21 +216,22 @@ class SyncWorker {
this._account.lastSyncCompletions = lastSyncCompletions
this._account.save()
- console.log('Syncworker: Completed sync cycle')
+ this._logger.info('Syncworker: Completed sync cycle')
if (afterSync === 'idle') {
return this._getIdleFolder()
.then((idleFolder) => this._conn.openBox(idleFolder.name))
- .then(() => console.log('SyncWorker: - Idling on inbox category'))
+ .then(() => this._logger.info('SyncWorker: Idling on inbox category'))
}
if (afterSync === 'close') {
- console.log('SyncWorker: - Closing connection');
+ this._logger.info('SyncWorker: Closing connection');
this.closeConnection()
return Promise.resolve()
}
- throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`)
+ this._logger.error({after_sync: afterSync}, `SyncWorker.onSyncDidComplete: Unknown afterSync behavior`)
+ throw new Error('SyncWorker.onSyncDidComplete: Unknown afterSync behavior')
}
isWaitingForNextSync() {
@@ -226,7 +240,7 @@ class SyncWorker {
scheduleNextSync() {
if (Date.now() - this._startTime > CLAIM_DURATION) {
- console.log("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool.");
+ this._logger.info("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool.");
this.cleanup();
this._onExpired();
return;
@@ -238,7 +252,10 @@ class SyncWorker {
if (interval) {
const target = this._lastSyncTime + interval;
- console.log(`SyncWorker: Account ${active ? 'active' : 'inactive'}. Next sync scheduled for ${new Date(target).toLocaleString()}`);
+ this._logger.info({
+ is_active: active,
+ next_sync: new Date(target).toLocaleString(),
+ }, `SyncWorker: Next sync scheduled`);
this._syncTimer = setTimeout(() => {
this.syncNow({reason: 'Scheduled'});
}, target - Date.now());
diff --git a/pm2-dev.yml b/pm2-dev.yml
index 313be4c0d..b8efea594 100644
--- a/pm2-dev.yml
+++ b/pm2-dev.yml
@@ -4,6 +4,7 @@ apps:
- script : packages/nylas-api/app.js
+ watch : ["packages"]
name : api
env :
PORT: 5100
@@ -12,25 +13,26 @@ apps:
GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com"
GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF"
GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback"
-
-
+ NODE_ENV: 'development'
- script : packages/nylas-sync/app.js
+ watch : ["packages"]
name : sync
env :
DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-
-
+ NODE_ENV: 'development'
- script : packages/nylas-dashboard/app.js
+ watch : ["packages"]
name : dashboard
env :
PORT: 5101
DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-
-
+ NODE_ENV: 'development'
- script : packages/nylas-message-processor/app.js
+ watch : ["packages"]
name : processor
env :
DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
+ NODE_ENV: 'development'
+