Do not insert empty keywords in FTS index

This commit is contained in:
mdecimus 2024-08-22 12:33:23 +02:00
parent bd4129e160
commit fe0ccb11bd
2 changed files with 21 additions and 11 deletions

View file

@ -95,11 +95,14 @@ impl<'x, T: Into<u8> + Display + Clone + std::fmt::Debug> FtsDocument<'x, T> {
} }
pub fn index_keyword(&mut self, field: Field<T>, text: impl Into<Cow<'x, str>>) { pub fn index_keyword(&mut self, field: Field<T>, text: impl Into<Cow<'x, str>>) {
self.parts.push(Text { let text = text.into();
field, if !text.is_empty() {
text: text.into(), self.parts.push(Text {
typ: Type::Keyword, field,
}); text,
typ: Type::Keyword,
});
}
} }
} }
@ -146,11 +149,14 @@ impl Store {
position += 10; position += 10;
} }
Type::Keyword => { Type::Keyword => {
let field = u8::from(text.field); let value = text.text.as_ref();
tokens if !value.is_empty() {
.entry(BitmapHash::new(text.text.as_ref())) let field = u8::from(text.field);
.or_default() tokens
.insert_keyword(TokenType::word(field)); .entry(BitmapHash::new(value))
.or_default()
.insert_keyword(TokenType::word(field));
}
} }
} }
} }
@ -268,6 +274,10 @@ impl Store {
hash[..len].copy_from_slice(&key[U32_LEN..U32_LEN + len]); hash[..len].copy_from_slice(&key[U32_LEN..U32_LEN + len]);
(hash, len as u8) (hash, len as u8)
} }
0 => {
// Temporary fix for empty keywords
(hash, 0)
}
invalid => { invalid => {
return Err(trc::Error::corrupted_key(key, None, trc::location!()) return Err(trc::Error::corrupted_key(key, None, trc::location!())
.ctx(trc::Key::Reason, "Invalid bitmap key length") .ctx(trc::Key::Reason, "Invalid bitmap key length")

View file

@ -250,7 +250,7 @@ impl<T: ResolveId> ValueClass<T> {
let serializer = serializer.write(account_id).write( let serializer = serializer.write(account_id).write(
hash.hash hash.hash
.get(0..std::cmp::min(hash.len as usize, 8)) .get(0..std::cmp::min(hash.len as usize, 8))
.unwrap(), .unwrap_or_default(),
); );
if hash.len >= 8 { if hash.len >= 8 {