mirror of
https://github.com/nodemailer/wildduck.git
synced 2024-12-26 09:50:47 +08:00
Started with searching from ES
This commit is contained in:
Andris Reinman
parent
5a9f584c74
commit
8a2a7b72ce
5 changed files with 132 additions and 70 deletions
|
|
@ -335,6 +335,8 @@ function indexingJob(esclient) {
|
|||
uid: messageData.uid,
|
||||
answered: messageData.flags ? messageData.flags.includes('\\Answered') : null,
|
||||
|
||||
ha: (messageData.attachments && messageData.attachments.length > 0) || false,
|
||||
|
||||
attachments:
|
||||
(messageData.attachments &&
|
||||
messageData.attachments.map(attachment =>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ const { nextPageCursorSchema, previousPageCursorSchema, pageNrSchema, sessSchema
|
|||
const { preprocessAttachments } = require('../data-url');
|
||||
const TaskHandler = require('../task-handler');
|
||||
const prepareSearchFilter = require('../prepare-search-filter');
|
||||
const { getMongoDBQuery } = require('../search-query');
|
||||
const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query');
|
||||
const { getClient } = require('./lib/elasticsearch');
|
||||
|
||||
const BimiHandler = require('../bimi-handler');
|
||||
|
||||
|
|
@ -584,6 +585,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti
|
|||
let query;
|
||||
|
||||
if (result.value.q) {
|
||||
let hasESFeatureFlag = await db.redis.sismember(`feature:indexing`, user.toString());
|
||||
if (hasESFeatureFlag) {
|
||||
// search from ElasticSearch
|
||||
|
||||
let searchQuery = await getElasticSearchQuery(db, user, result.value.q);
|
||||
|
||||
const esclient = getClient();
|
||||
|
||||
let searchResult = await esclient.search({
|
||||
index: config.elasticsearch.index,
|
||||
query: searchQuery,
|
||||
sort: { uid: 'desc' }
|
||||
});
|
||||
|
||||
console.log('ES RESULTS');
|
||||
console.log(util.inspect(searchResult, false, 22, true));
|
||||
}
|
||||
|
||||
filter = await getMongoDBQuery(db, user, result.value.q);
|
||||
query = result.value.q;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -63,6 +63,11 @@ const mappings = {
|
|||
type: 'boolean'
|
||||
},
|
||||
|
||||
// has attachments
|
||||
ha: {
|
||||
type: 'boolean'
|
||||
},
|
||||
|
||||
attachments: {
|
||||
type: 'nested',
|
||||
properties: {
|
||||
|
|
|
|||
|
|
@ -314,7 +314,7 @@ const getMongoDBQuery = async (db, user, queryStr) => {
|
|||
|
||||
return { user: false };
|
||||
};
|
||||
/*
|
||||
|
||||
const getElasticSearchQuery = async (db, user, queryStr) => {
|
||||
const parsed = parseSearchQuery(queryStr);
|
||||
|
||||
|
|
@ -330,9 +330,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
}
|
||||
};
|
||||
|
||||
let curNode = searchQuery;
|
||||
|
||||
let walkTree = async (node, curNode) => {
|
||||
let walkTree = async node => {
|
||||
if (Array.isArray(node)) {
|
||||
let branches = [];
|
||||
for (let entry of node) {
|
||||
|
|
@ -390,7 +388,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
};
|
||||
|
||||
if (node.text.negated) {
|
||||
// FIXME: negation support!
|
||||
branch = { bool: { must_not: branch.bool.should } };
|
||||
}
|
||||
|
||||
return branch;
|
||||
|
|
@ -401,23 +399,47 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
if (keyword) {
|
||||
let { value, negated } = node.keywords[keyword];
|
||||
switch (keyword) {
|
||||
case 'from':
|
||||
case 'subject':
|
||||
{
|
||||
let regex = escapeRegexStr(value);
|
||||
let branch = {
|
||||
headers: {
|
||||
$elemMatch: {
|
||||
key: keyword,
|
||||
value: {
|
||||
$regex: regex,
|
||||
$options: 'i'
|
||||
}
|
||||
match: {
|
||||
subject: {
|
||||
query: value,
|
||||
operator: 'and'
|
||||
}
|
||||
}
|
||||
};
|
||||
if (negated) {
|
||||
branch = { $not: branch };
|
||||
branch = { bool: { must_not: branch } };
|
||||
}
|
||||
branches.push(branch);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'from':
|
||||
{
|
||||
let branch = {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
[`from.name`]: {
|
||||
query: value,
|
||||
operator: 'and'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
term: {
|
||||
[`from.address`]: value
|
||||
}
|
||||
}
|
||||
],
|
||||
minimum_should_match: 1
|
||||
}
|
||||
};
|
||||
if (negated) {
|
||||
branch = { bool: { must_not: branch } };
|
||||
}
|
||||
branches.push(branch);
|
||||
}
|
||||
|
|
@ -425,24 +447,35 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
|
||||
case 'to':
|
||||
{
|
||||
let regex = escapeRegexStr(value);
|
||||
let branch = {
|
||||
bool: {
|
||||
should: [],
|
||||
minimum_should_match: 1
|
||||
}
|
||||
};
|
||||
|
||||
for (let toKey of ['to', 'cc', 'bcc']) {
|
||||
let branch = {
|
||||
headers: {
|
||||
$elemMatch: {
|
||||
key: toKey,
|
||||
value: {
|
||||
$regex: regex,
|
||||
$options: 'i'
|
||||
branch.bool.should.push(
|
||||
{
|
||||
match: {
|
||||
[`${toKey}.name`]: {
|
||||
query: value,
|
||||
operator: 'and'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
term: {
|
||||
[`${toKey}.address`]: value
|
||||
}
|
||||
}
|
||||
};
|
||||
if (negated) {
|
||||
branch = { $not: branch };
|
||||
}
|
||||
branches.push(branch);
|
||||
);
|
||||
}
|
||||
|
||||
if (negated) {
|
||||
branch = { bool: { must_not: branch } };
|
||||
}
|
||||
branches.push(branch);
|
||||
}
|
||||
break;
|
||||
|
||||
|
|
@ -468,9 +501,9 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
|
||||
let mailboxEntry = await db.database.collection('mailboxes').findOne(resolveQuery, { project: { _id: -1 } });
|
||||
|
||||
let branch = { mailbox: mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24)) };
|
||||
let branch = { term: { mailbox: (mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24))).toString() } };
|
||||
if (negated) {
|
||||
branch = { $not: branch };
|
||||
branch = { bool: { must_not: [branch] } };
|
||||
}
|
||||
branches.push(branch);
|
||||
|
||||
|
|
@ -481,9 +514,12 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
{
|
||||
value = (value || '').toString().trim();
|
||||
if (/^[0-9a-f]{24}$/i.test(value)) {
|
||||
let branch = { thread: new ObjectId(value) };
|
||||
let branch = { term: { thread: value } };
|
||||
if (negated) {
|
||||
branch = { $not: branch };
|
||||
branch = { bool: { must_not: [branch] } };
|
||||
}
|
||||
if (negated) {
|
||||
branch = { bool: { must_not: [branch] } };
|
||||
}
|
||||
branches.push(branch);
|
||||
}
|
||||
|
|
@ -493,7 +529,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
case 'has': {
|
||||
switch (value) {
|
||||
case 'attachment': {
|
||||
branches.push({ ha: true });
|
||||
branches.push({ term: { ha: true } });
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -506,39 +542,39 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
|
|||
};
|
||||
|
||||
if (parsed && parsed.length) {
|
||||
let filter = await walkTree(Array.isArray(parsed) ? { $and: parsed } : parsed);
|
||||
|
||||
let extras = { user };
|
||||
if (hasTextFilter) {
|
||||
extras.searchable = true;
|
||||
}
|
||||
|
||||
return Object.assign({ user: null }, filter, extras);
|
||||
let filter = await walkTree({ $and: parsed });
|
||||
searchQuery.bool.must = searchQuery.bool.must.concat(filter);
|
||||
}
|
||||
|
||||
return { user: false };
|
||||
return searchQuery;
|
||||
};
|
||||
*/
|
||||
|
||||
module.exports = { parseSearchQuery, getMongoDBQuery /*, getElasticSearchQuery*/ };
|
||||
module.exports = { parseSearchQuery, getMongoDBQuery, getElasticSearchQuery };
|
||||
|
||||
/*
|
||||
const util = require('util');
|
||||
if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') {
|
||||
const util = require('util'); // eslint-disable-line
|
||||
let main = () => {
|
||||
let db = require('./db'); // eslint-disable-line
|
||||
db.connect(() => {
|
||||
let run = async () => {
|
||||
let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)'];
|
||||
|
||||
let main = () => {
|
||||
let db = require('./db');
|
||||
db.connect(() => {
|
||||
let run = async () => {
|
||||
let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)'];
|
||||
for (let query of queries) {
|
||||
console.log('PARSED QUERY');
|
||||
console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true));
|
||||
console.log('MongoDB');
|
||||
console.log(util.inspect({ query, filter: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true));
|
||||
console.log('ElasticSearch');
|
||||
console.log(
|
||||
util.inspect({ query, filter: await getElasticSearchQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
for (let query of queries) {
|
||||
console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true));
|
||||
console.log(util.inspect({ query, parsed: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true));
|
||||
}
|
||||
};
|
||||
|
||||
run();
|
||||
});
|
||||
};
|
||||
main();
|
||||
*/
|
||||
run()
|
||||
.catch(err => console.error(err))
|
||||
.finally(() => process.exit());
|
||||
});
|
||||
};
|
||||
main();
|
||||
}
|
||||
|
|
|
|||
14
package.json
14
package.json
|
|
@ -26,7 +26,7 @@
|
|||
"ajv": "8.12.0",
|
||||
"chai": "4.3.7",
|
||||
"docsify-cli": "4.4.4",
|
||||
"eslint": "8.41.0",
|
||||
"eslint": "8.43.0",
|
||||
"eslint-config-nodemailer": "1.2.0",
|
||||
"eslint-config-prettier": "8.8.0",
|
||||
"grunt": "1.6.1",
|
||||
|
|
@ -35,7 +35,7 @@
|
|||
"grunt-mocha-test": "0.13.3",
|
||||
"grunt-shell-spawn": "0.4.0",
|
||||
"grunt-wait": "0.3.0",
|
||||
"imapflow": "1.0.128",
|
||||
"imapflow": "1.0.130",
|
||||
"mailparser": "3.6.4",
|
||||
"mocha": "10.2.0",
|
||||
"request": "2.88.2",
|
||||
|
|
@ -53,8 +53,8 @@
|
|||
"base32.js": "0.1.0",
|
||||
"bcryptjs": "2.4.3",
|
||||
"bson": "5.3.0",
|
||||
"bullmq": "3.14.0",
|
||||
"fido2-lib": "3.4.0",
|
||||
"bullmq": "3.15.8",
|
||||
"fido2-lib": "3.4.1",
|
||||
"gelf": "2.0.1",
|
||||
"generate-password": "1.7.0",
|
||||
"hash-wasm": "4.9.0",
|
||||
|
|
@ -64,7 +64,7 @@
|
|||
"iconv-lite": "0.6.3",
|
||||
"ioredfour": "1.2.0-ioredis-07",
|
||||
"ioredis": "5.3.2",
|
||||
"ipaddr.js": "2.0.1",
|
||||
"ipaddr.js": "2.1.0",
|
||||
"isemail": "3.2.0",
|
||||
"joi": "17.9.2",
|
||||
"js-yaml": "4.1.0",
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
"libmime": "5.2.1",
|
||||
"libqp": "2.0.1",
|
||||
"logic-query-parser": "0.0.5",
|
||||
"mailauth": "4.3.4",
|
||||
"mailauth": "4.4.0",
|
||||
"mailsplit": "5.4.0",
|
||||
"mobileconfig": "2.4.0",
|
||||
"mongo-cursor-pagination": "8.1.3",
|
||||
|
|
@ -82,7 +82,7 @@
|
|||
"msgpack5": "6.0.2",
|
||||
"node-forge": "1.3.1",
|
||||
"node-html-parser": "6.1.5",
|
||||
"nodemailer": "6.9.2",
|
||||
"nodemailer": "6.9.3",
|
||||
"npmlog": "7.0.1",
|
||||
"openpgp": "5.9.0",
|
||||
"pem-jwk": "2.0.0",
|
||||
|
|
|
|||
Loading…
Reference in a new issue