mirror of
https://github.com/Foundry376/Mailspring.git
synced 2024-09-25 17:56:06 +08:00
781080716c
Summary: This diff adds a new SearchIndexer class that each of the concrete indexer implementations register with. This new class uses the `isSearchIndexed` field in searchable classes to split indexing work into chunks. It then times how long it takes to index each chunk and schedules more work based on a target CPU percent. For example, if it takes 150 ms to index the last chunk of work and the target CPU fraction is 0.05 (i.e. 5% CPU use) then it will schedule the next increment of indexing 3 seconds in the future. Test Plan: Run locally, verify that indexing occurs in increments Reviewers: juan, evan Reviewed By: evan Differential Revision: https://phab.nylas.com/D3707
110 lines
3.2 KiB
JavaScript
110 lines
3.2 KiB
JavaScript
import {
|
|
DatabaseStore,
|
|
} from 'nylas-exports'
|
|
|
|
const FRACTION_CPU_AVAILABLE = 0.05;
|
|
const MAX_TIME_SLICE_MILLIS = 100;
|
|
const CHUNK_SIZE = 10;
|
|
const MIN_TIMEOUT = 100;
|
|
const MAX_TIMEOUT = 5 * 60 * 1000; // 5 minutes
|
|
|
|
export default class SearchIndexer {
|
|
constructor() {
|
|
this._searchableModels = {};
|
|
this._hasIndexingToDo = false;
|
|
this._lastTimeStart = null;
|
|
this._lastTimeStop = null;
|
|
}
|
|
|
|
registerSearchableModel(klass, indexCallback) {
|
|
this._searchableModels[klass.name] = {klass, cb: indexCallback};
|
|
}
|
|
|
|
unregisterSearchableModel(klass) {
|
|
delete this._searchableModels[klass.name];
|
|
}
|
|
|
|
async _getNewItemsToIndex() {
|
|
const results = await Promise.all(Object.keys(this._searchableModels).map((modelName) => {
|
|
const modelClass = this._searchableModels[modelName].klass;
|
|
const query = DatabaseStore.findAll(modelClass)
|
|
.whereAny([
|
|
modelClass.attributes.isSearchIndexed.equal(false),
|
|
modelClass.attributes.isSearchIndexed.equal(null),
|
|
])
|
|
.order(modelClass.attributes.id.ascending())
|
|
.limit(CHUNK_SIZE);
|
|
return query;
|
|
}));
|
|
return results.reduce((acc, curr) => acc.concat(curr), []);
|
|
}
|
|
|
|
_indexItems(items) {
|
|
for (const item of items) {
|
|
this._searchableModels[item.constructor.name].cb(item);
|
|
}
|
|
}
|
|
|
|
notifyHasIndexingToDo() {
|
|
if (this._hasIndexingToDo) {
|
|
return;
|
|
}
|
|
this._hasIndexingToDo = true;
|
|
this._scheduleRun();
|
|
}
|
|
|
|
_computeNextTimeout() {
|
|
if (!this._lastTimeStop || !this._lastTimeStart) {
|
|
return MIN_TIMEOUT;
|
|
}
|
|
const spanMillis = this._lastTimeStop.getTime() - this._lastTimeStart.getTime();
|
|
const multiplier = 1.0 / FRACTION_CPU_AVAILABLE;
|
|
return Math.min(Math.max(spanMillis * multiplier, MIN_TIMEOUT), MAX_TIMEOUT);
|
|
}
|
|
|
|
_scheduleRun() {
|
|
console.log(`SearchIndexer: setting timeout for ${this._computeNextTimeout()} ms`);
|
|
setTimeout(() => this.run(), this._computeNextTimeout());
|
|
}
|
|
|
|
run() {
|
|
if (!this._hasIndexingToDo) {
|
|
return;
|
|
}
|
|
|
|
const start = new Date();
|
|
let current = new Date();
|
|
let firstIter = true;
|
|
let numItemsIndexed = 0;
|
|
|
|
const indexNextChunk = (unindexedItems) => {
|
|
console.info('unindexedItems:', unindexedItems);
|
|
if (firstIter) {
|
|
this._lastTimeStart = start;
|
|
firstIter = false;
|
|
}
|
|
|
|
if (unindexedItems.length === 0) {
|
|
this._hasIndexingToDo = false;
|
|
this._lastTimeStop = new Date();
|
|
console.info(`Finished indexing ${numItemsIndexed} items, took ${current.getTime() - start.getTime()} ms`);
|
|
return;
|
|
}
|
|
|
|
this._indexItems(unindexedItems);
|
|
numItemsIndexed += unindexedItems.length;
|
|
current = new Date();
|
|
|
|
if (current.getTime() - start.getTime() <= MAX_TIME_SLICE_MILLIS) {
|
|
this._getNewItemsToIndex().then(indexNextChunk);
|
|
return;
|
|
}
|
|
|
|
this._lastTimeStop = new Date();
|
|
console.info(`SearchIndexer: Finished indexing ${numItemsIndexed} items, took ${current.getTime() - start.getTime()} ms`);
|
|
this._scheduleRun();
|
|
};
|
|
this._getNewItemsToIndex().then(indexNextChunk);
|
|
}
|
|
}
|