Decode base64 encoded attachments before storing to DB

config/attachments.toml

@@ -0,0 +1,9 @@
+# Attachment storage options
+
+# For now there's only a single option for attachment storage
+type="gridstore"
+bucket="attachments"
+
+# If true then decodes base64 encoded attachments to binary before storing to DB.
+# Decoding base64 attachments expects consistent line length and default base64 alphabet
+decodeBase64=true

config/default.toml

@@ -37,9 +37,7 @@ bugsnagCode=""
     #secret="a secret cat"
 
 [attachments]
-    # For now there's only a single option for attachment storage
+# @include "attachments.toml"
-    type="gridstore"
-    bucket="attachments"
 
 [log]
     level="silly"

imap-core/lib/indexer/indexer.js

@@ -193,18 +193,24 @@ class Indexer {
                     attachmentId = mimeTree.attachmentMap[node.attachmentId];
                 }
 
-                let attachmentStream = this.attachmentStorage.createReadStream(attachmentId);
+                return this.attachmentStorage.get(attachmentId, (err, attachmentData) => {
+                    if (err) {
+                        return res.emit('error', err);
+                    }
 
-                attachmentStream.once('error', err => {
+                    let attachmentStream = this.attachmentStorage.createReadStream(attachmentId, attachmentData);
-                    res.errored = err;
+
+                    attachmentStream.once('error', err => {
+                        // res.errored = err;
+                        res.emit('error', err);
+                    });
+
+                    attachmentStream.once('end', () => finalize());
+
+                    attachmentStream.pipe(res, {
+                        end: false
+                    });
                 });
-
-                attachmentStream.once('end', () => finalize());
-
-                attachmentStream.pipe(res, {
-                    end: false
-                });
-                return;
             }
 
             let pos = 0;
@@ -399,6 +405,7 @@ class Indexer {
                     magic: maildata.magic,
                     contentType,
                     transferEncoding,
+                    lineCount: node.lineCount,
                     body: node.body
                 });
 

lib/api/messages.js

@@ -885,10 +885,21 @@ module.exports = (db, server, messageHandler) => {
                     'Content-Type': attachmentData.contentType || 'application/octet-stream'
                 });
 
-                let attachmentStream = messageHandler.attachmentStorage.createReadStream(attachmentId);
+                let decode = true;
+
+                if (attachmentData.metadata.decoded) {
+                    attachmentData.metadata.decoded = false;
+                    decode = false;
+                }
+
+                let attachmentStream = messageHandler.attachmentStorage.createReadStream(attachmentId, attachmentData);
 
                 attachmentStream.once('error', err => res.emit('error', err));
 
+                if (!decode) {
+                    return attachmentStream.pipe(res);
+                }
+
                 if (attachmentData.transferEncoding === 'base64') {
                     attachmentStream.pipe(new libbase64.Decoder()).pipe(res);
                 } else if (attachmentData.transferEncoding === 'quoted-printable') {

lib/attachment-storage.js

@@ -36,8 +36,8 @@ class AttachmentStorage {
         });
     }
 
-    createReadStream(id) {
+    createReadStream(id, attachmentData) {
-        return this.storage.createReadStream(id);
+        return this.storage.createReadStream(id, attachmentData);
     }
 
     deleteMany(ids, magic, callback) {
@@ -68,7 +68,15 @@ class AttachmentStorage {
         let algo = 'sha256';
 
         if (!cryptoAsync) {
-            setImmediate(() => callback(null, crypto.createHash(algo).update(input).digest('hex')));
+            setImmediate(() =>
+                callback(
+                    null,
+                    crypto
+                        .createHash(algo)
+                        .update(input)
+                        .digest('hex')
+                )
+            );
             return;
         }
 

lib/attachments/gridstore-storage.js

@@ -3,7 +3,8 @@
 const GridFSBucket = require('mongodb').GridFSBucket;
 const libbase64 = require('libbase64');
 
-const FEATURE_DECODE_ATTACHMENTS = false;
+// Set to false to disable base64 decoding feature
+const FEATURE_DECODE_ATTACHMENTS = true;
 
 class GridstoreStorage {
     constructor(options) {
@@ -87,8 +88,18 @@ class GridstoreStorage {
             }
 
             if (lineLen && lineLen <= 998) {
-                metadata.decoded = true;
+                if (attachment.body.length === lineLen && lineLen < 76) {
-                metadata.lineLen = lineLen;
+                    lineLen = 76;
+                }
+
+                // check if expected line count matches with attachment line count
+                let expectedLineCount = Math.ceil(attachment.body.length / (lineLen + 2));
+
+                // allow 1 line shift
+                if (attachment.lineCount >= expectedLineCount - 1 && attachment.lineCount <= expectedLineCount + 1) {
+                    metadata.decoded = true;
+                    metadata.lineLen = lineLen;
+                }
             }
         }
 
@@ -162,8 +173,23 @@ class GridstoreStorage {
         tryStore();
     }
 
-    createReadStream(id) {
+    createReadStream(id, attachmentData) {
-        return this.gridstore.openDownloadStream(id);
+        let stream = this.gridstore.openDownloadStream(id);
+        if (attachmentData.metadata.decoded) {
+            let encoder = new libbase64.Encoder({
+                lineLength: attachmentData.metadata.lineLen
+            });
+
+            stream.once('error', err => {
+                // pass error forward
+                encoder.emit('error', err);
+            });
+            stream.pipe(encoder);
+
+            return encoder;
+        }
+
+        return stream;
     }
 
     delete(id, magic, callback) {
@@ -254,7 +280,7 @@ class GridstoreStorage {
                 }
 
                 // delete file entry first
-                this.gridfs.collection('attachments.files').deleteOne({
+                this.gridfs.collection(this.bucketName + '.files').deleteOne({
                     _id: attachment._id,
                     // make sure that we do not delete a message that is already re-used
                     'metadata.c': 0,
@@ -265,7 +291,7 @@ class GridstoreStorage {
                     }
 
                     // delete data chunks
-                    this.gridfs.collection('attachments.chunks').deleteMany({
+                    this.gridfs.collection(this.bucketName + '.chunks').deleteMany({
                         files_id: attachment._id
                     }, err => {
                         if (err) {
@@ -283,7 +309,7 @@ class GridstoreStorage {
     }
 
     cleanupGarbage(id, next) {
-        this.gridfs.collection('attachments.files').findOne({
+        this.gridfs.collection(this.bucketName + '.files').findOne({
             _id: id
         }, (err, file) => {
             if (err) {
@@ -295,7 +321,7 @@ class GridstoreStorage {
             }
 
             // orphaned attachment, delete data chunks
-            this.gridfs.collection('attachments.chunks').deleteMany({
+            this.gridfs.collection(this.bucketName + '.chunks').deleteMany({
                 files_id: id
             }, (err, info) => {
                 if (err) {