Mailspring/script/bootstrap
Ben Gotow a14a5212ac feat(transactions): Explicit (and faster) database transactions
Summary:
Until now, we've been hiding transactions beneath the surface. When you call persistModel, you're implicitly creating a transaction.
You could explicitly create them with `atomically`..., but there were several critical problems that are fixed in this diff:

- Calling persistModel / unpersistModel within a transaction could cause the DatabaseStore to trigger. This could result in other parts of the app making queries /during/
  the transaction, potentially before the COMMIT occurred and saved the changes. The new, explicit inTransaction syntax holds all changes until after COMMIT and then triggers.

- Calling atomically and then calling persistModel inside that resulted in us having to check whether a transaction was present and was gross.

- Many parts of the code ran extensive logic inside a promise chained within `atomically`:

  BAD:

```
  DatabaseStore.atomically =>
   DatabaseStore.persistModel(draft) =>
     GoMakeANetworkRequestThatReturnsAPromise
```

OVERWHELMINGLY BETTER:

```
  DatabaseStore.inTransaction (t) =>
     t.persistModel(draft)
  .then =>
    GoMakeANetworkRequestThatReturnsAPromise
```

Having explicit transactions also puts us on equal footing with Sequelize and other ORMs. Note that you /have/ to call DatabaseStore.inTransaction (t) =>. There is no other way to access the methods that let you alter the database. :-)

Other changes:
- This diff removes Message.labels and the Message-Labels table. We weren't using Message-level labels anywhere, and the table could grow very large.
- This diff changes the page size during initial sync from 250 => 200 in an effort to make transactions a bit faster.

Test Plan: Run tests!

Reviewers: juan, evan

Reviewed By: juan, evan

Differential Revision: https://phab.nylas.com/D2353
2015-12-17 11:46:05 -08:00

258 lines
9.6 KiB
JavaScript
Executable file

#!/usr/bin/env node
var fs = require('fs');
var verifyRequirements = require('./utils/verify-requirements');
var safeExec = require('./utils/child-process-wrapper.js').safeExec;
var path = require('path');
var userHomeDirectory = (process.platform === 'win32') ? process.env.USERPROFILE : process.env.HOME;
var appPackageJSON = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', 'package.json')));
var targetElectronVersion = appPackageJSON['electronVersion'];
var targetPlatform = require('os').platform();
// Executes an array of commands in series
function executeCommands(commands, done, index) {
index = (index == undefined ? 0 : index);
if (index < commands.length) {
var command = commands[index];
if (command.message)
console.log(command.message);
var options = null;
if (typeof command !== 'string') {
options = command.options;
command = command.command;
}
safeExec(command, options, executeCommands.bind(this, commands, done, index + 1));
}
else
done(null);
}
function printArgs(args) {
out = "";
for(key in args) {
out += "--"+key+"="+args[key]+" ";
}
return out;
}
function makeSqlite3Command() {
var nodeGypPath = '"' + path.resolve(__dirname, '..', 'build', 'node_modules', 'npm', 'node_modules', '.bin', 'node-gyp') + '"';
if (targetPlatform == "win32") {
// As of Electron 0.29.2, all windows machines, even if they're 64 bit,
// return "ia32" as the arch
var targetArch = "ia32"
} else {
var targetArch = require('os').arch();
}
// Use our local version of npm (npm 3x) to build sqlite
var npmPath = '"' + path.resolve(__dirname, '..', 'build', 'node_modules', '.bin', 'npm') + '"';
return npmPath + " install https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep.tar.gz --ignore-scripts && cd node_modules/sqlite3 && "+nodeGypPath+" configure rebuild --target="+targetElectronVersion+" --arch="+targetArch+" --target_platform="+targetPlatform+" --dist-url=https://atom.io/download/atom-shell --module_name=node_sqlite3 --module_path=../lib/binding/node-v46-"+targetPlatform+"-"+targetArch
}
function bootstrap() {
var apmDestinationPath = path.resolve(__dirname, '..', 'apm');
if (!fs.existsSync(apmDestinationPath))
fs.mkdirSync(apmDestinationPath);
if (!fs.existsSync(path.join(apmDestinationPath, 'node_modules')))
fs.mkdirSync(path.join(apmDestinationPath, 'node_modules'));
var apmPath = '"' + path.resolve(__dirname, '..', 'apm', 'node_modules', 'atom-package-manager', 'bin', 'apm') + '"';
var apmFlags = process.env.JANKY_SHA1 || process.argv.indexOf('--no-color') !== -1 ? ' --no-color' : '';
var npmFlags = ' --userconfig=' + '"' + path.resolve('.npmrc') + '" ';
var gruntPath = '"' + path.resolve(__dirname, '..', 'build', 'node_modules', '.bin', 'grunt') + '"';
var packagesToDedupe = ['fs-plus', 'humanize-plus', 'roaster', 'season', 'grim'];
// use the system version of npm to install build folder dependencies, including our
// own copy of NPM 3.x, which we'll use for subsequent commands
var buildInstallCommand = 'npm' + npmFlags + 'install';
var buildInstallOptions = {cwd: path.resolve(__dirname, '..', 'build')};
var npmPath = '"' + path.resolve(__dirname, '..', 'build', 'node_modules', '.bin', 'npm') + '"';
// Use our local version of npm in ./build to install apm. This ensures it gets a
// flat dependency tree.
var npmInstallApmCommand = npmPath + npmFlags + '--target=0.10.40 ' + 'install';
var npmInstallApmOptions = {cwd: apmDestinationPath};
var npmDedupeNpmCommand = npmPath + npmFlags + '--target=0.10.40 ' + 'dedupe';
var npmDedupeNpmOptions = {cwd: path.join(apmDestinationPath, 'node_modules', 'npm')};
var sqlite3Command = makeSqlite3Command();
var apmInstallCommand = apmPath + ' install' + apmFlags;
var apmEnv = JSON.parse(JSON.stringify(process.env))
apmEnv['ATOM_ELECTRON_VERSION'] = targetElectronVersion;
apmEnv['ATOM_HOME'] = path.join(userHomeDirectory, '.nylas');
var apmDedupeCommand = apmPath + ' dedupe' + apmFlags;
var semverOptions = {
cwd: path.resolve(__dirname, '..', 'apm', 'node_modules', 'atom-package-manager'),
env: apmEnv
};
if (process.argv.indexOf('--no-quiet') === -1) {
buildInstallCommand += ' --loglevel error';
npmInstallApmCommand += ' --loglevel error';
apmInstallCommand += ' --loglevel error';
apmDedupeCommand += ' --quiet';
npmDedupeNpmOptions.ignoreStderr = true;
npmDedupeNpmOptions.ignoreStdout = true;
buildInstallOptions.ignoreStdout = true;
npmInstallApmOptions.ignoreStdout = true;
}
// apm ships with 32-bit node so make sure its native modules are compiled
// for a 32-bit target architecture
if (process.env.JANKY_SHA1 && process.platform === 'win32')
npmInstallApmCommand += ' --arch=ia32';
m1 = "\n---> Installing N1 build tools\n"
m1 += " This goes inside the `build` folder and runs `npm install`\n"
m1 += " It will use the system `npm` to bootstrap our own N1 npm.\n"
m1 += " Our build tools (like Grunt) need to be compiled against Node via `npm`.\n"
m1 += " Everything else needs to be compiled against Chromium with `apm`.\n\n"
m1 += " $ "+buildInstallCommand+" "+printArgs(buildInstallOptions)+"\n"
m2 = "\n\n---> Installing apm\n"
m2 += " This installs apm via N1's `npm`\n"
m2 += " We use this local apm copy to install all N1 dependencies & packages\n\n"
m2 += " $ "+npmInstallApmCommand+" "+printArgs(npmInstallApmOptions)+"\n"
m2a = "\n\n---> Flattening apm package tree\n"
m2a += " This runs `npm dedupe` on apm's `npm` dependency.\n"
m2a += " We use this to prevent paths over 260 characters on Windows.\n\n"
m2a += " $ "+npmDedupeNpmCommand+" "+printArgs(npmDedupeNpmOptions)+"\n"
m3 = "\n\n---> Cleaning apm via `apm clean`\n"
m4 = "\n\n---> Installing N1 dependencies & packages via `apm install`\n\n"
m4 += " $ "+apmInstallCommand+"\n"
m5 = "\n\n---> De-duping packages `apm dedupe`\n\n"
m5 += " $ "+apmDedupeCommand+" "+packagesToDedupe.join(' ')+"\n"
m6 = "\n\n---> Request version `apm dedupe`\n\n"
m6 += " $ "+apmDedupeCommand+" request semver "+printArgs(semverOptions)+"\n"
m7 = "\n\n---> Getting latest Electron\n\n"
var gruntCmd = ""
var downloadElectronCmd = gruntPath + " download-electron --gruntfile build/Gruntfile.coffee"
m7 += " $ "+downloadElectronCmd
var integrationCommand = npmPath + npmFlags + 'install';
var integrationOptions = {cwd: path.resolve(__dirname, '..', 'spec_integration')};
m8 = "\n\n---> Installing integration test modules\n\n"
m8 += " $ "+integrationCommand+" "+printArgs(integrationOptions)+"\n"
var commands = [
{
command: buildInstallCommand,
message: m1,
options: buildInstallOptions
},
{
command: gruntPath + " add-nylas-build-resources --gruntfile build/Gruntfile.coffee",
message: "",
options: {}
}
]
process.chdir(path.dirname(__dirname));
executeCommands(commands, function() {
var commands = [
{
command: npmInstallApmCommand,
message: m2,
options: npmInstallApmOptions
},
{
command: npmDedupeNpmCommand,
message: m2a,
options: npmDedupeNpmOptions
},
{
command: apmPath + ' clean' + apmFlags,
options: { env: apmEnv },
message: m3
}
];
// we need this because we don't put our modules in node_modules and npm
// install doesn't find them. Run APM install on each package directory manually.
internalPackagesDir = path.resolve(__dirname, '..', 'internal_packages');
internalPackages = fs.readdirSync('internal_packages');
internalPackages.forEach(function(dir) {
var dirPackageJSONPath = path.join(internalPackagesDir, dir, 'package.json');
// On windows and linux, invoking the apm command is very slow even when there are no
// dependencies. Make it faster by not calling unless we find there are deps.
if (fs.existsSync(dirPackageJSONPath)) {
var dirPackageJSON = JSON.parse(fs.readFileSync(dirPackageJSONPath));
if (dirPackageJSON.dependencies && (Object.keys(dirPackageJSON.dependencies).length > 0)) {
commands.push({
command: apmInstallCommand,
message: "Installing dependencies for "+dir,
options: {
cwd: path.join(internalPackagesDir, dir),
env: apmEnv
}
});
}
}
});
commands = commands.concat([
{
command: apmInstallCommand,
options: { env: apmEnv },
message: m4
},
{
command: apmDedupeCommand + ' ' + packagesToDedupe.join(' '),
options: { env: apmEnv },
message: m5
},
{
command: apmDedupeCommand + ' request semver',
options: semverOptions,
message: m6
},
{
command: downloadElectronCmd,
message: m7
},
{
command: integrationCommand,
options: integrationOptions,
message: m8
},
{
command: sqlite3Command,
message: "Building sqlite3 with command: "+sqlite3Command
}
]);
process.chdir(path.dirname(__dirname));
executeCommands(commands, function() {
console.log("---------------------------------------------");
console.log("script/bootstrap completed successfully. You can start\nN1 via ./N1.sh --dev, or run tests with ./N1.sh --test");
process.exit();
});
});
}
verifyRequirements(function(error, successMessage) {
if (error) {
console.log(error);
process.exit(1);
}
console.log(successMessage);
bootstrap();
});