Reuse TantivySearcher across queries via searcher pool

Author: chenghuichen

paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexer.java

@@ -35,12 +35,6 @@ public class TantivyFullTextGlobalIndexer implements GlobalIndexer {
     private final Map<String, ArchiveLayout> layoutCache = new ConcurrentHashMap<>();
     private final TantivySearcherPool searcherPool;
 
-    public TantivyFullTextGlobalIndexer() {
-        this(
-                new TantivySearcherPool(
-                        TantivyFullTextIndexOptions.SEARCHER_POOL_MAX_SIZE.defaultValue()));
-    }
-
     public TantivyFullTextGlobalIndexer(TantivySearcherPool searcherPool) {
         this.searcherPool = searcherPool;
     }

paimon-tantivy/paimon-tantivy-index/src/test/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexTest.java

@@ -41,8 +41,10 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -70,11 +72,15 @@ private static boolean isNativeAvailable() {
 
     private FileIO fileIO;
     private Path indexPath;
+    private Map<String, ArchiveLayout> layoutCache;
+    private TantivySearcherPool pool;
 
     @BeforeEach
     public void setup() {
         fileIO = new LocalFileIO();
         indexPath = new Path(tempDir.toString());
+        layoutCache = new ConcurrentHashMap<>();
+        pool = new TantivySearcherPool(4);
     }
 
     @AfterEach
@@ -111,6 +117,11 @@ private List<GlobalIndexIOMeta> toIOMetas(List<ResultEntry> results, Path path)
                 new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta()));
     }
 
+    private TantivyFullTextGlobalIndexReader createReader(
+            GlobalIndexFileReader fileReader, List<GlobalIndexIOMeta> metas) {
+        return new TantivyFullTextGlobalIndexReader(fileReader, metas, layoutCache, pool);
+    }
+
     @Test
     public void testEndToEnd() throws IOException {
         GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
@@ -127,12 +138,7 @@ public void testEndToEnd() throws IOException {
         List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
         GlobalIndexFileReader fileReader = createFileReader();
 
-        try (TantivyFullTextGlobalIndexReader reader =
-                new TantivyFullTextGlobalIndexReader(
-                        fileReader,
-                        metas,
-                        new java.util.concurrent.ConcurrentHashMap<>(),
-                        new TantivySearcherPool(4))) {
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
             FullTextSearch search = new FullTextSearch("paimon", 10, "text");
             Optional<ScoredGlobalIndexResult> searchResult = reader.visitFullTextSearch(search);
             assertThat(searchResult).isPresent();
@@ -164,12 +170,7 @@ public void testSearchNoResults() throws IOException {
         List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
         GlobalIndexFileReader fileReader = createFileReader();
 
-        try (TantivyFullTextGlobalIndexReader reader =
-                new TantivyFullTextGlobalIndexReader(
-                        fileReader,
-                        metas,
-                        new java.util.concurrent.ConcurrentHashMap<>(),
-                        new TantivySearcherPool(4))) {
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
             FullTextSearch search = new FullTextSearch("nonexistent", 10, "text");
             Optional<ScoredGlobalIndexResult> searchResult = reader.visitFullTextSearch(search);
             assertThat(searchResult).isPresent();
@@ -194,12 +195,7 @@ public void testNullFieldSkipped() throws IOException {
         List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
         GlobalIndexFileReader fileReader = createFileReader();
 
-        try (TantivyFullTextGlobalIndexReader reader =
-                new TantivyFullTextGlobalIndexReader(
-                        fileReader,
-                        metas,
-                        new java.util.concurrent.ConcurrentHashMap<>(),
-                        new TantivySearcherPool(4))) {
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
             FullTextSearch search = new FullTextSearch("paimon", 10, "text");
             Optional<ScoredGlobalIndexResult> searchResult = reader.visitFullTextSearch(search);
             assertThat(searchResult).isPresent();
@@ -243,12 +239,7 @@ public void testLargeDataset() throws IOException {
         List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
         GlobalIndexFileReader fileReader = createFileReader();
 
-        try (TantivyFullTextGlobalIndexReader reader =
-                new TantivyFullTextGlobalIndexReader(
-                        fileReader,
-                        metas,
-                        new java.util.concurrent.ConcurrentHashMap<>(),
-                        new TantivySearcherPool(4))) {
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
             // Search for the special keyword — should match every 10th doc
             FullTextSearch search = new FullTextSearch("special_keyword", 1000, "text");
             Optional<ScoredGlobalIndexResult> searchResult = reader.visitFullTextSearch(search);
@@ -277,12 +268,7 @@ public void testLimitRespected() throws IOException {
         List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
         GlobalIndexFileReader fileReader = createFileReader();
 
-        try (TantivyFullTextGlobalIndexReader reader =
-                new TantivyFullTextGlobalIndexReader(
-                        fileReader,
-                        metas,
-                        new java.util.concurrent.ConcurrentHashMap<>(),
-                        new TantivySearcherPool(4))) {
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
             // Limit to 5 results
             FullTextSearch search = new FullTextSearch("paimon", 5, "text");
             Optional<ScoredGlobalIndexResult> searchResult = reader.visitFullTextSearch(search);
@@ -293,9 +279,38 @@ public void testLimitRespected() throws IOException {
         }
     }
 
+    @Test
+    public void testPoolReuse() throws IOException {
+        GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
+        TantivyFullTextGlobalIndexWriter writer = new TantivyFullTextGlobalIndexWriter(fileWriter);
+        writer.write(BinaryString.fromString("Apache Paimon streaming lake"));
+        writer.write(BinaryString.fromString("Tantivy full-text search"));
+
+        List<ResultEntry> results = writer.finish();
+        List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
+        GlobalIndexFileReader fileReader = createFileReader();
+        FullTextSearch search = new FullTextSearch("paimon", 10, "text");
+
+        // First query: pool miss, searcher is loaded and returned to pool on close.
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
+            Optional<ScoredGlobalIndexResult> result = reader.visitFullTextSearch(search);
+            assertThat(result).isPresent();
+            assertThat(result.get().results().contains(0L)).isTrue();
+        }
+
+        // Second query: pool hit, reuses the same searcher — results must be identical.
+        try (TantivyFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
+            Optional<ScoredGlobalIndexResult> result = reader.visitFullTextSearch(search);
+            assertThat(result).isPresent();
+            assertThat(result.get().results().getLongCardinality()).isEqualTo(1);
+            assertThat(result.get().results().contains(0L)).isTrue();
+        }
+    }
+
     @Test
     public void testViaIndexer() throws IOException {
-        TantivyFullTextGlobalIndexer indexer = new TantivyFullTextGlobalIndexer();
+        TantivyFullTextGlobalIndexer indexer =
+                new TantivyFullTextGlobalIndexer(new TantivySearcherPool(0));
 
         GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
         TantivyFullTextGlobalIndexWriter writer =