Merge pull request #36598 from vespa-engine/toregge/add-document-id-provider-interface-used-by-document-id-dfw

Add IDocumentIdProvider, used by DocumentIdDFW to get document id string

Author: boeker

searchcore/src/tests/proton/documentdb/configurer_test.cpp

@@ -223,7 +223,8 @@ Fixture::initViewSet(ViewSet &views)
                                                  views._service.write().field_writer(), views._service.write().shared(), views._hwInfo);
     auto summaryMgr = make_shared<SummaryManager>
             (_summaryExecutor, search::LogDocumentStore::Config(), search::GrowStrategy(), BASE_DIR,
-             TuneFileSummary(), views._fileHeaderContext,views._noTlSyncer, search::IBucketizer::SP());
+             TuneFileSummary(), views._fileHeaderContext,views._noTlSyncer, search::IBucketizer::SP(),
+             std::shared_ptr<const search::IDocumentIdProvider>());
     auto metaStore = make_shared<DocumentMetaStoreContext>(make_shared<bucketdb::BucketDBOwner>());
     auto indexWriter = std::make_shared<IndexWriter>(indexMgr);
     auto attrWriter = std::make_shared<AttributeWriter>(attrMgr);

searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp

@@ -91,6 +91,7 @@ SummaryManager::SummarySetup::
 SummarySetup(const std::string & baseDir, const SummaryConfig & summaryCfg,
              const JuniperrcConfig & juniperCfg,
              search::IAttributeManager::SP attributeMgr, search::IDocumentStore::SP docStore,
+             std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
              std::shared_ptr<const DocumentTypeRepo> repo,
              const search::index::Schema& schema)
     : _docsumWriter(),
@@ -99,6 +100,7 @@ SummarySetup(const std::string & baseDir, const SummaryConfig & summaryCfg,
       _juniperConfig(),
       _attributeMgr(std::move(attributeMgr)),
       _docStore(std::move(docStore)),
+      _document_id_provider(std::move(document_id_provider)),
       _repo(std::move(repo))
 {
     _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, _wordFolder.get());
@@ -126,6 +128,9 @@ SummaryManager::SummarySetup::createDocsumStore()
     return std::make_unique<DocumentStoreAdapter>(*_docStore, *_repo);
 }
 
+std::shared_ptr<const search::IDocumentIdProvider> SummaryManager::SummarySetup::get_document_id_provider() const noexcept {
+    return _document_id_provider;
+}
 
 ISummaryManager::ISummarySetup::SP
 SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg,
@@ -134,16 +139,18 @@ SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg,
                                    const search::index::Schema& schema)
 {
     return std::make_shared<SummarySetup>(_baseDir, summaryCfg,
-                                          juniperCfg, attributeMgr, _docStore, repo, schema);
+                                          juniperCfg, attributeMgr, _docStore, _document_id_provider, repo, schema);
 }
 
 SummaryManager::SummaryManager(vespalib::Executor &shared_executor, const LogDocumentStore::Config & storeConfig,
                                const search::GrowStrategy & growStrategy, const std::string &baseDir,
                                const TuneFileSummary &tuneFileSummary,
                                const FileHeaderContext &fileHeaderContext, search::transactionlog::SyncProxy &tlSyncer,
-                               search::IBucketizer::SP bucketizer)
+                               search::IBucketizer::SP bucketizer,
+                               std::shared_ptr<const search::IDocumentIdProvider> document_id_provider)
     : _baseDir(baseDir),
-      _docStore()
+      _docStore(),
+      _document_id_provider(std::move(document_id_provider))
 {
     _docStore = std::make_shared<LogDocumentStore>(shared_executor, baseDir, storeConfig, growStrategy, tuneFileSummary,
                                                    fileHeaderContext, tlSyncer, std::move(bucketizer));

searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h

@@ -28,13 +28,15 @@ class SummaryManager : public ISummaryManager
         std::unique_ptr<juniper::Juniper>     _juniperConfig;
         search::IAttributeManager::SP         _attributeMgr;
         search::IDocumentStore::SP            _docStore;
+        std::shared_ptr<const search::IDocumentIdProvider>       _document_id_provider;
         const std::shared_ptr<const document::DocumentTypeRepo>  _repo;
     public:
         SummarySetup(const std::string & baseDir,
                      const SummaryConfig & summaryCfg,
                      const JuniperrcConfig & juniperCfg,
                      search::IAttributeManager::SP attributeMgr,
                      search::IDocumentStore::SP docStore,
+                     std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
                      std::shared_ptr<const document::DocumentTypeRepo> repo,
                      const search::index::Schema& schema);
 
@@ -45,11 +47,13 @@ class SummaryManager : public ISummaryManager
 
         const search::IAttributeManager * getAttributeManager() const override { return _attributeMgr.get(); }
         const juniper::Juniper * getJuniper() const override { return _juniperConfig.get(); }
+        [[nodiscard]] std::shared_ptr<const search::IDocumentIdProvider> get_document_id_provider() const noexcept override;
     };
 
 private:
     std::string               _baseDir;
-    std::shared_ptr<search::IDocumentStore> _docStore;
+    std::shared_ptr<search::IDocumentStore>            _docStore;
+    std::shared_ptr<const search::IDocumentIdProvider> _document_id_provider;
 
 public:
     using SP = std::shared_ptr<SummaryManager>;
@@ -60,7 +64,8 @@ class SummaryManager : public ISummaryManager
                    const search::TuneFileSummary &tuneFileSummary,
                    const search::common::FileHeaderContext &fileHeaderContext,
                    search::transactionlog::SyncProxy &tlSyncer,
-                   std::shared_ptr<search::IBucketizer> bucketizer);
+                   std::shared_ptr<search::IBucketizer> bucketizer,
+                   std::shared_ptr<const search::IDocumentIdProvider> document_id_provider);
     ~SummaryManager() override;
 
     void putDocument(uint64_t syncToken, search::DocumentIdT lid, const document::Document & doc);

searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp

@@ -17,6 +17,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow,
                           const search::common::FileHeaderContext &fileHeaderContext,
                           search::transactionlog::SyncProxy &tlSyncer,
                           IBucketizerSP bucketizer,
+                          std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
                           std::shared_ptr<SummaryManager::SP> result)
     : proton::initializer::InitializerTask(),
       _grow(grow),
@@ -28,6 +29,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow,
       _fileHeaderContext(fileHeaderContext),
       _tlSyncer(tlSyncer),
       _bucketizer(std::move(bucketizer)),
+      _document_id_provider(std::move(document_id_provider)),
       _result(std::move(result))
 { }
 
@@ -42,7 +44,7 @@ SummaryManagerInitializer::run()
     MemoryUsageLogger::log("start load document store", _subDbName);
     *_result = std::make_shared<SummaryManager>
                (_shared_executor, _storeCfg, _grow, _baseDir,
-                _tuneFile, _fileHeaderContext, _tlSyncer, _bucketizer);
+                _tuneFile, _fileHeaderContext, _tlSyncer, _bucketizer, _document_id_provider);
     EventLogger::loadDocumentStoreComplete(_subDbName, timer.elapsed());
     MemoryUsageLogger::log("finish load document store", _subDbName);
 }

searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h

@@ -24,6 +24,7 @@ class SummaryManagerInitializer : public initializer::InitializerTask
     const search::common::FileHeaderContext &_fileHeaderContext;
     search::transactionlog::SyncProxy       &_tlSyncer;
     const IBucketizerSP                      _bucketizer;
+    std::shared_ptr<const search::IDocumentIdProvider> _document_id_provider;
     std::shared_ptr<SummaryManager::SP>      _result;
 
 public:
@@ -39,6 +40,7 @@ class SummaryManagerInitializer : public initializer::InitializerTask
                               const search::common::FileHeaderContext & fileHeaderContext,
                               search::transactionlog::SyncProxy &tlSyncer,
                               IBucketizerSP bucketizer,
+                              std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
                               std::shared_ptr<SummaryManager::SP> result);
     ~SummaryManagerInitializer() override;
     void run() override;

searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp

@@ -231,13 +231,15 @@ createSummaryManagerInitializer(const search::LogDocumentStore::Config & storeCf
                                 const AllocStrategy& alloc_strategy,
                                 const search::TuneFileSummary &tuneFile,
                                 search::IBucketizer::SP bucketizer,
+                                std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
                                 std::shared_ptr<SummaryManager::SP> result) const
 {
     GrowStrategy grow = alloc_strategy.get_grow_strategy();
     std::string baseDir(_baseDir + "/summary");
     return std::make_shared<SummaryManagerInitializer>
         (grow, baseDir, getSubDbName(), _writeService.shared(),
-         storeCfg, tuneFile, _fileHeaderContext, _tlSyncer, std::move(bucketizer), std::move(result));
+         storeCfg, tuneFile, _fileHeaderContext, _tlSyncer, std::move(bucketizer),
+         std::move(document_id_provider), std::move(result));
 }
 
 void
@@ -321,10 +323,13 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser
                                                           configSnapshot.getTuneFileDocumentDBSP()->_attr,
                                                           result->writableResult().writableDocumentMetaStore());
     result->addDocumentMetaStoreInitTask(dmsInitTask);
+    auto dms = result->result().documentMetaStore()->documentMetaStore();
+    std::shared_ptr<const search::IDocumentIdProvider> document_id_provider; // No document id provider yet.
     auto summaryTask = createSummaryManagerInitializer(createStoreConfig(configSnapshot.getStoreConfig(), _subDbType),
                                                        alloc_strategy,
                                                        configSnapshot.getTuneFileDocumentDBSP()->_summary,
-                                                       result->result().documentMetaStore()->documentMetaStore(),
+                                                       std::move(dms),
+                                                       std::move(document_id_provider),
                                                        result->writableResult().writableSummaryManager());
     result->addDependency(summaryTask);
     summaryTask->addDependency(dmsInitTask);

searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h

@@ -167,6 +167,7 @@ class StoreOnlyDocSubDB : public DocSubDB
                                     const AllocStrategy& alloc_strategy,
                                     const search::TuneFileSummary &tuneFile,
                                     search::IBucketizer::SP bucketizer,
+                                    std::shared_ptr<const search::IDocumentIdProvider> document_id_provider,
                                     std::shared_ptr<SummaryManager::SP> result) const;
 
     void setupSummaryManager(SummaryManager::SP summaryManager);

searchlib/src/vespa/searchlib/common/i_document_id_provider.h

@@ -0,0 +1,20 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+namespace search {
+
+/*
+ * This class provides a document id string view for the document with the given lid (local document id).
+ * An empty string view indicates that the document id is not available.
+ */
+class IDocumentIdProvider {
+public:
+    virtual ~IDocumentIdProvider() = default;
+   [[nodiscard]] virtual std::string_view get_document_id_string_view(uint32_t lid) const noexcept = 0;
+};
+
+}

searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp

@@ -5,6 +5,7 @@
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/repo/newconfigbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchlib/common/i_document_id_provider.h>
 #include <vespa/searchlib/common/matching_elements.h>
 #include <vespa/searchsummary/docsummary/docsumstate.h>
 #include <vespa/searchsummary/docsummary/docsum_store_document.h>
@@ -14,7 +15,7 @@
 #include <vespa/searchsummary/docsummary/summary_elements_selector.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/gtest/gtest.h>
-#include <iostream>
+#include <map>
 #include <memory>
 
 using document::Document;
@@ -40,6 +41,7 @@ namespace {
 
 const int32_t          doc_type_id   = 787121340;
 const std::string doc_type_name = "test";
+const std::string provided_document_id("id:provided:test::1");
 
 std::unique_ptr<const DocumentTypeRepo>
 make_doc_type_repo()
@@ -55,21 +57,51 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback {
     std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &) override { abort(); }
 };
 
+class MyDocumentIdProvider : public search::IDocumentIdProvider {
+    std::map<uint32_t, std::string> _ids;
+public:
+    MyDocumentIdProvider();
+    ~MyDocumentIdProvider() override;
+    [[nodiscard]] std::string_view get_document_id_string_view(uint32_t lid) const noexcept override;
+};
+
+MyDocumentIdProvider::MyDocumentIdProvider()
+    : search::IDocumentIdProvider(),
+      _ids()
+{
+    _ids.emplace(1, provided_document_id);
+}
+
+MyDocumentIdProvider::~MyDocumentIdProvider() = default;
+
+std::string_view MyDocumentIdProvider::get_document_id_string_view(uint32_t lid) const noexcept {
+    auto itr = _ids.find(lid);
+    if (itr != _ids.end()) {
+        return itr->second;
+    } else {
+        return {};
+    }
+}
+
+}
+
 class DocumentIdDFWTest : public ::testing::Test
 {
     std::string                        _field_name;
     vespalib::Memory                        _field_name_view;
     std::unique_ptr<ResultConfig>           _result_config;
     std::unique_ptr<const DocumentTypeRepo> _repo;
     const DocumentType*                     _document_type;
+    std::shared_ptr<const search::IDocumentIdProvider> _document_id_provider;
 
 protected:
     DocumentIdDFWTest();
     ~DocumentIdDFWTest() override;
 
     std::unique_ptr<IDocsumStoreDocument> make_docsum_store_document(const std::string &id);
-    vespalib::Slime write(const IDocsumStoreDocument* doc);
+    Slime write(uint32_t lid, const IDocsumStoreDocument* doc);
     vespalib::Memory get_field_name_view() const noexcept { return _field_name_view; }
+    void enable_document_id_provider() { _document_id_provider = std::make_shared<MyDocumentIdProvider>(); }
 };
 
 DocumentIdDFWTest::DocumentIdDFWTest()
@@ -78,7 +110,8 @@ DocumentIdDFWTest::DocumentIdDFWTest()
       _field_name_view(_field_name.data(), _field_name.size()),
       _result_config(std::make_unique<ResultConfig>()),
       _repo(make_doc_type_repo()),
-      _document_type(_repo->getDocumentType(doc_type_name))
+      _document_type(_repo->getDocumentType(doc_type_name)),
+      _document_id_provider()
 {
     auto* cfg = _result_config->addResultClass("default", 0);
     cfg->addConfigEntry(_field_name);
@@ -96,41 +129,60 @@ DocumentIdDFWTest::make_docsum_store_document(const std::string& id)
 }
 
 vespalib::Slime
-DocumentIdDFWTest::write(const IDocsumStoreDocument* doc)
+DocumentIdDFWTest::write(uint32_t lid, const IDocsumStoreDocument* doc)
 {
     Slime slime;
     SlimeInserter top_inserter(slime);
     Cursor & docsum = top_inserter.insertObject();
     ObjectInserter field_inserter(docsum, _field_name_view);
-    DocumentIdDFW writer;
+    DocumentIdDFW writer(_document_id_provider);
     MyGetDocsumsStateCallback callback;
     GetDocsumsState state(callback);
-    writer.insert_field(0, doc, state, ElementIds::select_all(), field_inserter);
+    writer.insert_field(lid, doc, state, ElementIds::select_all(), field_inserter);
     return slime;
 }
 
 TEST_F(DocumentIdDFWTest, insert_document_id)
 {
     std::string id("id::test::0");
     auto doc = make_docsum_store_document(id);
-    auto slime = write(doc.get());
+    auto slime = write(1, doc.get());
     EXPECT_TRUE(slime.get()[get_field_name_view()].valid());
     EXPECT_EQ(id, slime.get()[get_field_name_view()].asString().make_string());
 }
 
 TEST_F(DocumentIdDFWTest, insert_document_id_no_document_doc)
 {
     auto doc = std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>());
-    auto slime = write(doc.get());
+    auto slime = write(1, doc.get());
     EXPECT_FALSE(slime.get()[get_field_name_view()].valid());
 }
 
 TEST_F(DocumentIdDFWTest, insert_document_id_no_docsum_store_doc)
 {
-    auto slime = write(nullptr);
+    auto slime = write(1, nullptr);
     EXPECT_FALSE(slime.get()[get_field_name_view()].valid());
 }
 
+TEST_F(DocumentIdDFWTest, insert_document_id_no_document_doc_but_document_id_provider)
+{
+    enable_document_id_provider();
+    auto doc = std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>());
+    auto slime = write(1, doc.get());
+    EXPECT_TRUE(slime.get()[get_field_name_view()].valid());
+    EXPECT_EQ(provided_document_id, slime.get()[get_field_name_view()].asString().make_string());
+    slime = write(2, doc.get());
+    EXPECT_FALSE(slime.get()[get_field_name_view()].valid());
+}
+
+TEST_F(DocumentIdDFWTest, insert_document_id_no_docsum_store_doc_but_document_id_provider)
+{
+    enable_document_id_provider();
+    auto slime = write(1, nullptr);
+    EXPECT_TRUE(slime.get()[get_field_name_view()].valid());
+    EXPECT_EQ(provided_document_id, slime.get()[get_field_name_view()].asString().make_string());
+    slime = write(2, nullptr);
+    EXPECT_FALSE(slime.get()[get_field_name_view()].valid());
 }
 
 GTEST_MAIN_RUN_ALL_TESTS()

searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp

@@ -129,7 +129,7 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const std::string& field_na
             throw_if_nullptr(fieldWriter, command);
         }
     } else if (command == command::documentid) {
-        fieldWriter = std::make_unique<DocumentIdDFW>();
+        fieldWriter = std::make_unique<DocumentIdDFW>(_env.get_document_id_provider());
     } else {
         throw IllegalArgumentException("Unknown command '" + command + "'.");
     }