From 8a2a7b72ce03d7a04b24796faece2f96534ea06b Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 22 Jun 2023 12:37:20 +0300 Subject: [PATCH] Started with searching from ES --- indexer.js | 2 + lib/api/messages.js | 21 +++++- lib/ensure-es-index.js | 5 ++ lib/search-query.js | 160 +++++++++++++++++++++++++---------------- package.json | 14 ++-- 5 files changed, 132 insertions(+), 70 deletions(-) diff --git a/indexer.js b/indexer.js index 5eec996f..fc4bbec8 100644 --- a/indexer.js +++ b/indexer.js @@ -335,6 +335,8 @@ function indexingJob(esclient) { uid: messageData.uid, answered: messageData.flags ? messageData.flags.includes('\\Answered') : null, + ha: (messageData.attachments && messageData.attachments.length > 0) || false, + attachments: (messageData.attachments && messageData.attachments.map(attachment => diff --git a/lib/api/messages.js b/lib/api/messages.js index 87b43fba..e29a3694 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -21,7 +21,8 @@ const { nextPageCursorSchema, previousPageCursorSchema, pageNrSchema, sessSchema const { preprocessAttachments } = require('../data-url'); const TaskHandler = require('../task-handler'); const prepareSearchFilter = require('../prepare-search-filter'); -const { getMongoDBQuery } = require('../search-query'); +const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query'); +const { getClient } = require('./lib/elasticsearch'); const BimiHandler = require('../bimi-handler'); @@ -584,6 +585,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let query; if (result.value.q) { + let hasESFeatureFlag = await db.redis.sismember(`feature:indexing`, user.toString()); + if (hasESFeatureFlag) { + // search from ElasticSearch + + let searchQuery = await getElasticSearchQuery(db, user, result.value.q); + + const esclient = getClient(); + + let searchResult = await esclient.search({ + index: config.elasticsearch.index, + query: searchQuery, + sort: { uid: 'desc' } + }); + + console.log('ES RESULTS'); + console.log(util.inspect(searchResult, false, 22, true)); + } + filter = await getMongoDBQuery(db, user, result.value.q); query = result.value.q; } else { diff --git a/lib/ensure-es-index.js b/lib/ensure-es-index.js index b96a377d..2efac611 100644 --- a/lib/ensure-es-index.js +++ b/lib/ensure-es-index.js @@ -63,6 +63,11 @@ const mappings = { type: 'boolean' }, + // has attachments + ha: { + type: 'boolean' + }, + attachments: { type: 'nested', properties: { diff --git a/lib/search-query.js b/lib/search-query.js index fa7c1bd8..5028d108 100644 --- a/lib/search-query.js +++ b/lib/search-query.js @@ -314,7 +314,7 @@ const getMongoDBQuery = async (db, user, queryStr) => { return { user: false }; }; -/* + const getElasticSearchQuery = async (db, user, queryStr) => { const parsed = parseSearchQuery(queryStr); @@ -330,9 +330,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { } }; - let curNode = searchQuery; - - let walkTree = async (node, curNode) => { + let walkTree = async node => { if (Array.isArray(node)) { let branches = []; for (let entry of node) { @@ -390,7 +388,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }; if (node.text.negated) { - // FIXME: negation support! + branch = { bool: { must_not: branch.bool.should } }; } return branch; @@ -401,23 +399,47 @@ const getElasticSearchQuery = async (db, user, queryStr) => { if (keyword) { let { value, negated } = node.keywords[keyword]; switch (keyword) { - case 'from': case 'subject': { - let regex = escapeRegexStr(value); let branch = { - headers: { - $elemMatch: { - key: keyword, - value: { - $regex: regex, - $options: 'i' - } + match: { + subject: { + query: value, + operator: 'and' } } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: branch } }; + } + branches.push(branch); + } + break; + + case 'from': + { + let branch = { + bool: { + should: [ + { + match: { + [`from.name`]: { + query: value, + operator: 'and' + } + } + }, + { + term: { + [`from.address`]: value + } + } + ], + minimum_should_match: 1 + } + }; + if (negated) { + branch = { bool: { must_not: branch } }; } branches.push(branch); } @@ -425,24 +447,35 @@ const getElasticSearchQuery = async (db, user, queryStr) => { case 'to': { - let regex = escapeRegexStr(value); + let branch = { + bool: { + should: [], + minimum_should_match: 1 + } + }; + for (let toKey of ['to', 'cc', 'bcc']) { - let branch = { - headers: { - $elemMatch: { - key: toKey, - value: { - $regex: regex, - $options: 'i' + branch.bool.should.push( + { + match: { + [`${toKey}.name`]: { + query: value, + operator: 'and' } } + }, + { + term: { + [`${toKey}.address`]: value + } } - }; - if (negated) { - branch = { $not: branch }; - } - branches.push(branch); + ); } + + if (negated) { + branch = { bool: { must_not: branch } }; + } + branches.push(branch); } break; @@ -468,9 +501,9 @@ const getElasticSearchQuery = async (db, user, queryStr) => { let mailboxEntry = await db.database.collection('mailboxes').findOne(resolveQuery, { project: { _id: -1 } }); - let branch = { mailbox: mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24)) }; + let branch = { term: { mailbox: (mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24))).toString() } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: [branch] } }; } branches.push(branch); @@ -481,9 +514,12 @@ const getElasticSearchQuery = async (db, user, queryStr) => { { value = (value || '').toString().trim(); if (/^[0-9a-f]{24}$/i.test(value)) { - let branch = { thread: new ObjectId(value) }; + let branch = { term: { thread: value } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: [branch] } }; + } + if (negated) { + branch = { bool: { must_not: [branch] } }; } branches.push(branch); } @@ -493,7 +529,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { case 'has': { switch (value) { case 'attachment': { - branches.push({ ha: true }); + branches.push({ term: { ha: true } }); break; } } @@ -506,39 +542,39 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }; if (parsed && parsed.length) { - let filter = await walkTree(Array.isArray(parsed) ? { $and: parsed } : parsed); - - let extras = { user }; - if (hasTextFilter) { - extras.searchable = true; - } - - return Object.assign({ user: null }, filter, extras); + let filter = await walkTree({ $and: parsed }); + searchQuery.bool.must = searchQuery.bool.must.concat(filter); } - return { user: false }; + return searchQuery; }; -*/ -module.exports = { parseSearchQuery, getMongoDBQuery /*, getElasticSearchQuery*/ }; +module.exports = { parseSearchQuery, getMongoDBQuery, getElasticSearchQuery }; -/* -const util = require('util'); +if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') { + const util = require('util'); // eslint-disable-line + let main = () => { + let db = require('./db'); // eslint-disable-line + db.connect(() => { + let run = async () => { + let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)']; -let main = () => { - let db = require('./db'); - db.connect(() => { - let run = async () => { - let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)']; + for (let query of queries) { + console.log('PARSED QUERY'); + console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true)); + console.log('MongoDB'); + console.log(util.inspect({ query, filter: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)); + console.log('ElasticSearch'); + console.log( + util.inspect({ query, filter: await getElasticSearchQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true) + ); + } + }; - for (let query of queries) { - console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true)); - console.log(util.inspect({ query, parsed: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)); - } - }; - - run(); - }); -}; -main(); -*/ + run() + .catch(err => console.error(err)) + .finally(() => process.exit()); + }); + }; + main(); +} diff --git a/package.json b/package.json index ccf81d93..0750f205 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "ajv": "8.12.0", "chai": "4.3.7", "docsify-cli": "4.4.4", - "eslint": "8.41.0", + "eslint": "8.43.0", "eslint-config-nodemailer": "1.2.0", "eslint-config-prettier": "8.8.0", "grunt": "1.6.1", @@ -35,7 +35,7 @@ "grunt-mocha-test": "0.13.3", "grunt-shell-spawn": "0.4.0", "grunt-wait": "0.3.0", - "imapflow": "1.0.128", + "imapflow": "1.0.130", "mailparser": "3.6.4", "mocha": "10.2.0", "request": "2.88.2", @@ -53,8 +53,8 @@ "base32.js": "0.1.0", "bcryptjs": "2.4.3", "bson": "5.3.0", - "bullmq": "3.14.0", - "fido2-lib": "3.4.0", + "bullmq": "3.15.8", + "fido2-lib": "3.4.1", "gelf": "2.0.1", "generate-password": "1.7.0", "hash-wasm": "4.9.0", @@ -64,7 +64,7 @@ "iconv-lite": "0.6.3", "ioredfour": "1.2.0-ioredis-07", "ioredis": "5.3.2", - "ipaddr.js": "2.0.1", + "ipaddr.js": "2.1.0", "isemail": "3.2.0", "joi": "17.9.2", "js-yaml": "4.1.0", @@ -73,7 +73,7 @@ "libmime": "5.2.1", "libqp": "2.0.1", "logic-query-parser": "0.0.5", - "mailauth": "4.3.4", + "mailauth": "4.4.0", "mailsplit": "5.4.0", "mobileconfig": "2.4.0", "mongo-cursor-pagination": "8.1.3", @@ -82,7 +82,7 @@ "msgpack5": "6.0.2", "node-forge": "1.3.1", "node-html-parser": "6.1.5", - "nodemailer": "6.9.2", + "nodemailer": "6.9.3", "npmlog": "7.0.1", "openpgp": "5.9.0", "pem-jwk": "2.0.0",