wildduck/lib/ensure-es-index.js

'use strict';

const assert = require('assert');

const analyzer = {
    htmlStripAnalyzer: {
        type: 'custom',
        tokenizer: 'standard',
        filter: ['lowercase'],
        char_filter: ['html_strip']
    },
    filenameSearch: {
        tokenizer: 'filename',
        filter: ['lowercase']
    },
    filenameIndex: {
        tokenizer: 'filename',
        filter: ['lowercase', 'edgeNgram']
    }
};

const tokenizer = {
    filename: {
        pattern: '[^\\p{L}\\d]+',
        type: 'pattern'
    }
};

const filter = {
    edgeNgram: {
        side: 'front',
        max_gram: 20,
        min_gram: 1,
        type: 'edge_ngram'
    }
};

const mappings = {
    // user ID / ObjectId
    user: {
        type: 'keyword',
        ignore_above: 24
    },

    // mailbox folder ID / ObjectId
    mailbox: {
        type: 'keyword',
        ignore_above: 24
    },

    // Thread ID / ObjectId
    thread: {
        type: 'keyword',
        ignore_above: 24
    },

    // Folder UID
    uid: {
        type: 'long'
    },

    answered: {
        type: 'boolean'
    },

    // has attachments
    ha: {
        type: 'boolean'
    },

    attachments: {
        type: 'nested',
        properties: {
            cid: {
                type: 'keyword',
                ignore_above: 128
            },
            contentType: {
                type: 'keyword',
                ignore_above: 128
            },
            size: {
                type: 'long'
            },
            filename: {
                type: 'text',
                analyzer: 'filenameIndex',
                search_analyzer: 'filenameSearch'
            },
            id: {
                type: 'keyword',
                ignore_above: 128
            },
            disposition: {
                type: 'keyword',
                ignore_above: 128
            }
        }
    },

    bcc: {
        properties: {
            address: {
                type: 'keyword',
                ignore_above: 256
            },
            name: {
                type: 'text'
            }
        }
    },

    cc: {
        properties: {
            address: {
                type: 'keyword',
                ignore_above: 256
            },
            name: {
                type: 'text'
            }
        }
    },

    // Time when stored
    created: {
        type: 'date'
    },

    // Internal Date
    idate: {
        type: 'date'
    },

    // Header Date
    hdate: {
        type: 'date'
    },

    draft: {
        type: 'boolean'
    },

    flagged: {
        type: 'boolean'
    },

    flags: {
        type: 'keyword',
        ignore_above: 128
    },

    from: {
        properties: {
            address: {
                type: 'keyword',
                ignore_above: 256
            },
            name: {
                type: 'text'
            }
        }
    },

    headers: {
        type: 'nested',
        properties: {
            key: {
                type: 'keyword',
                ignore_above: 256
            },
            value: {
                type: 'text'
            }
        }
    },

    inReplyTo: {
        type: 'keyword',
        ignore_above: 998
    },

    msgid: {
        type: 'keyword',
        ignore_above: 998
    },

    replyTo: {
        properties: {
            address: {
                type: 'keyword',
                ignore_above: 256
            },
            name: {
                type: 'text'
            }
        }
    },

    size: {
        type: 'long'
    },

    subject: {
        type: 'text'
    },

    to: {
        properties: {
            name: {
                type: 'text'
            },
            address: {
                type: 'keyword',
                ignore_above: 256
            }
        }
    },

    unseen: {
        type: 'boolean'
    },

    html: {
        type: 'text',
        analyzer: 'htmlStripAnalyzer'
    },

    text: {
        type: 'text'
    },

    type: {
        type: 'constant_keyword',
        value: 'email'
    },

    modseq: {
        type: 'long'
    }
};

/**
 * Function to either create or update an index to match the definition
 * @param {Object} client ElasticSearch client object
 * @param {String} index Index name
 */
const ensureIndex = async (client, index, opts) => {
    const { mappings, analyzer, tokenizer, filter, aliases } = opts;

    let indexExistsRes = await client.indices.exists({ index });
    let indexExists = indexExistsRes && indexExistsRes.body;

    if (!indexExists || !indexExists) {
        // create new

        let indexOpts = {
            mappings: { properties: mappings }
        };

        if (analyzer || tokenizer || filter) {
            indexOpts.settings = {
                analysis: {
                    analyzer,
                    tokenizer,
                    filter
                }
            };
        }

        if (aliases) {
            indexOpts.aliases = aliases;
        }

        let createResultRes = await client.indices.create({ index, body: indexOpts });
        let createResult = createResultRes && createResultRes.body;

        assert(createResult && createResult.acknowledged);
        return { created: true };
    } else {
        let indexDataRes = await client.indices.get({ index });
        let indexData = indexDataRes && indexDataRes.body;

        if (!indexData || !indexData[index]) {
            throw new Error('Missing index data');
        }

        let changes = {};

        if (analyzer || tokenizer || filter) {
            // compare settings and update if needed
            let analysisData = (indexData[index].settings && indexData[index].settings.index && indexData[index].settings.index.analysis) || {};
            let missingAnalyzers = {};
            for (let key of Object.keys(analyzer || {})) {
                if (!analysisData.analyzer || !analysisData.analyzer[key]) {
                    missingAnalyzers[key] = analyzer[key];
                }
            }

            // found missing analyzers, update settings
            if (Object.keys(missingAnalyzers).length) {
                // index needs to be closed when changing analyser settings
                let closeResultRes = await client.indices.close({ index });
                let closeResult = closeResultRes && closeResultRes.body;

                assert(closeResult && closeResult.acknowledged);

                try {
                    let updateResultRes = await client.indices.putSettings({
                        index,
                        body: {
                            settings: {
                                analysis: {
                                    analyzer,
                                    tokenizer,
                                    filter
                                }
                            }
                        }
                    });
                    let updateResult = updateResultRes && updateResultRes.body;

                    assert(updateResult && updateResult.acknowledged);
                    changes.analyzers = true;
                } finally {
                    // try to open even if update failed
                    let openResultRes = await client.indices.open({ index });
                    let openResult = openResultRes && openResultRes.body;

                    assert(openResult && openResult.acknowledged);
                }
            }
        }

        // Compare mappings and add missing
        let storedMappings = (indexData[index].mappings && indexData[index].mappings.properties) || {};
        let missingMappings = {};
        for (let key of Object.keys(mappings)) {
            if (!storedMappings[key]) {
                missingMappings[key] = mappings[key];
            }
        }
        // Add missing mappings if needed
        if (Object.keys(missingMappings).length) {
            try {
                const updateResponseRes = await client.indices.putMapping({
                    index,
                    body: { properties: missingMappings }
                });
                const updateResponse = updateResponseRes && updateResponseRes.body;

                assert(updateResponse && updateResponse.acknowledged);
                changes.mappings = true;
            } catch (err) {
                // other than that update everything succeeded, so ignore for now
            }
        }

        if (!Object.keys(changes).length) {
            return { exists: true };
        } else {
            return { updated: true, changes };
        }
    }
};

module.exports = {
    ensureIndex: async (client, index) => await ensureIndex(client, index, { mappings, analyzer, tokenizer, filter })
};