update tantivy (#6006)

* update tantivy

* allow deprecated

* new TopDocs api

Author: PSeitz

quickwit/Cargo.lock

@@ -346,7 +346,7 @@ quickwit-serve = { path = "quickwit-serve" }
 quickwit-storage = { path = "quickwit-storage" }
 quickwit-telemetry = { path = "quickwit-telemetry" }
 
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "25d44fcec8", default-features = false, features = [
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "618e3bd", default-features = false, features = [
   "lz4-compression",
   "mmap",
   "quickwit",

quickwit/Cargo.toml

@@ -263,20 +263,32 @@ fn extract_term_set_query_fields(
     Ok(visitor.term_dict_fields_to_warm_up)
 }
 
+/// Converts a `prefix` term into the equivalent term range.
+///
+/// The resulting range is `[prefix, next_prefix)`, that is:
+/// - start bound: `Included(prefix)`
+/// - end bound: `Excluded(next lexicographic term after the prefix)`
+///
+/// "abc"    -> start: "abc", end: "abd" (excluded)
+/// "ab\xFF" -> start: "ab\xFF", end: "ac" (excluded)
+/// "\xFF\xFF" -> start: "\xFF\xFF", end: Unbounded
 fn prefix_term_to_range(prefix: Term) -> (Bound<Term>, Bound<Term>) {
-    let mut end_bound = prefix.serialized_term().to_vec();
-    while !end_bound.is_empty() {
-        let last_byte = end_bound.last_mut().unwrap();
+    // Start from the given prefix and try to find the successor
+    let mut end_bound = prefix.clone();
+    let mut end_bound_value_bytes = prefix.serialized_value_bytes().to_vec();
+    while !end_bound_value_bytes.is_empty() {
+        let last_byte = end_bound_value_bytes.last_mut().unwrap();
         if *last_byte != u8::MAX {
             *last_byte += 1;
-            return (
-                Bound::Included(prefix),
-                Bound::Excluded(Term::wrap(end_bound)),
-            );
+            // The last non-`u8::MAX` byte incremented
+            // gives us the exclusive upper bound.
+            end_bound.set_bytes(&end_bound_value_bytes);
+            return (Bound::Included(prefix), Bound::Excluded(end_bound));
         }
-        end_bound.pop();
+        // pop u8::MAX byte and try next
+        end_bound_value_bytes.pop();
     }
-    // prefix is something like [255, 255, ..]
+    // All bytes were `u8::MAX`: there is no successor, so the upper bound is unbounded.
     (Bound::Included(prefix), Bound::Unbounded)
 }
 

quickwit/quickwit-doc-mapper/src/query_builder.rs

@@ -839,7 +839,8 @@ mod tests {
             let documents_left = searcher
                 .search(
                     &tantivy::query::AllQuery,
-                    &tantivy::collector::TopDocs::with_limit(result_docs.len() + 1),
+                    &tantivy::collector::TopDocs::with_limit(result_docs.len() + 1)
+                        .order_by_score(),
                 )?
                 .into_iter()
                 .map(|(_, doc_address)| {

quickwit/quickwit-indexing/src/actors/merge_executor.rs

@@ -562,6 +562,7 @@ impl SpanReaderPlugin for JaegerService {
     }
 }
 
+#[allow(deprecated)]
 fn extract_term(term_bytes: &[u8]) -> String {
     tantivy::Term::wrap(term_bytes)
         .value()

quickwit/quickwit-jaeger/src/lib.rs

@@ -306,14 +306,17 @@ pub async fn single_node_search(
 #[cfg(any(test, feature = "testsuite"))]
 #[macro_export]
 macro_rules! encode_term_for_test {
-    ($field:expr, $value:expr) => {
-        ::tantivy::schema::Term::from_field_text(
-            ::tantivy::schema::Field::from_field_id($field),
-            $value,
-        )
-        .serialized_term()
-        .to_vec()
-    };
+    ($field:expr, $value:expr) => {{
+        #[allow(deprecated)]
+        {
+            ::tantivy::schema::Term::from_field_text(
+                ::tantivy::schema::Field::from_field_id($field),
+                $value,
+            )
+            .serialized_term()
+            .to_vec()
+        }
+    }};
     ($value:expr) => {
         encode_term_for_test!(0, $value)
     };

quickwit/quickwit-search/src/lib.rs

@@ -206,6 +206,7 @@ pub fn jobs_to_leaf_requests(
 
 /// Apply a leaf list terms on a single split.
 #[instrument(skip_all, fields(split_id = split.split_id))]
+#[allow(deprecated)]
 async fn leaf_list_terms_single_split(
     searcher_context: &SearcherContext,
     search_request: &ListTermsRequest,
@@ -308,6 +309,7 @@ fn term_from_data(field: Field, field_type: &FieldType, data: &[u8]) -> Term {
     term
 }
 
+#[allow(deprecated)]
 fn term_to_data(field: Field, field_type: &FieldType, field_value: &[u8]) -> Vec<u8> {
     let mut term = Term::from_field_bool(field, false);
     term.clear_with_type(field_type.value_type());

quickwit/quickwit-search/src/list_terms.rs

@@ -1646,11 +1646,12 @@ async fn test_single_node_range_queries() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[allow(deprecated)]
 fn collect_str_terms(response: LeafListTermsResponse) -> Vec<String> {
     response
         .terms
         .into_iter()
-        .map(|term| Term::wrap(term).value().as_str().unwrap().to_string())
+        .map(|term| Term::wrap(&term).value().as_str().unwrap().to_string())
         .collect()
 }