nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-09-11 07:34:48 +08:00
Code Issues Projects Releases Packages Wiki Activity

use sha256 hash as the _id for attachments

Browse source
This commit is contained in:
Andris Reinman 2017-08-09 11:11:38 +03:00
parent 6748a5c876
commit 1657673c02
3 changed files with 72 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -197,6 +197,7 @@ Shard the following collections by these keys:
sh.enableSharding('wildduck');
sh.shardCollection('wildduck.messages', { mailbox: 1, uid: 1 });
sh.shardCollection('wildduck.threads', { user: 'hashed' });
// attachment _id is a sha256 hash of attachment contents
sh.shardCollection('wildduck.attachments.files', { _id: 'hashed' });
sh.shardCollection('wildduck.attachments.chunks', { files_id: 'hashed' });
```

6
indexes.yaml
View file

@ -270,12 +270,6 @@ indexes:
name: attachment_id_hashed
key:
_id: hashed
- collection: attachments.files
type: gridfs # index applies to gridfs database
index:
name: attachment_hash
key:
metadata.h: hashed
- collection: attachments.files
type: gridfs # index applies to gridfs database
index:

127
lib/attachments/gridstore-storage.js
View file

@ -1,6 +1,5 @@
'use strict';
const ObjectID = require('mongodb').ObjectID;
const GridFSBucket = require('mongodb').GridFSBucket;
class GridstoreStorage {
@ -28,69 +27,85 @@ class GridstoreStorage {
transferEncoding: attachmentData.metadata.transferEncoding,
length: attachmentData.length,
count: attachmentData.metadata.c,
hash: attachmentData.metadata.h,
hash: attachmentData._id,
metadata: attachmentData.metadata
});
});
}
create(attachment, hash, callback) {
this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({
'metadata.h': hash
}, {
$inc: {
'metadata.c': 1,
'metadata.m': attachment.magic
hash = Buffer.from(hash, 'hex');
let returned = false;
let retried = false;
let id = hash;
let metadata = {
m: attachment.magic,
c: 1,
transferEncoding: attachment.transferEncoding
};
Object.keys(attachment.metadata || {}).forEach(key => {
if (!(key in attachment.metadata)) {
metadata[key] = attachment.metadata[key];
}
}, {
returnOriginal: false
}, (err, result) => {
if (err) {
return callback(err);
}
if (result && result.value) {
return callback(null, result.value._id);
}
let returned = false;
let id = new ObjectID();
let metadata = {
h: hash,
m: attachment.magic,
c: 1,
transferEncoding: attachment.transferEncoding
};
Object.keys(attachment.metadata || {}).forEach(key => {
if (!(key in attachment.metadata)) {
metadata[key] = attachment.metadata[key];
}
});
let store = this.gridstore.openUploadStreamWithId(id, null, {
contentType: attachment.contentType,
metadata
});
store.once('error', err => {
if (returned) {
return;
}
returned = true;
callback(err);
});
store.once('finish', () => {
if (returned) {
return;
}
returned = true;
return callback(null, id);
});
store.end(attachment.body);
});
let tryStore = () => {
this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({
_id: hash
}, {
$inc: {
'metadata.c': 1,
'metadata.m': attachment.magic
}
}, {
returnOriginal: false
}, (err, result) => {
if (err) {
return callback(err);
}
if (result && result.value) {
// already exists
return callback(null, result.value._id);
}
// try to insert it
let store = this.gridstore.openUploadStreamWithId(id, null, {
contentType: attachment.contentType,
metadata
});
store.once('error', err => {
if (returned) {
return;
}
if (err.code === 11000) {
// most probably a race condition, try again
if (!retried) {
retried = true;
return setTimeout(tryStore, 10);
}
}
returned = true;
callback(err);
});
store.once('finish', () => {
if (returned) {
return;
}
returned = true;
return callback(null, id);
});
store.end(attachment.body);
});
};
tryStore();
}
createReadStream(id) {