Merge pull request #36366 from vespa-engine/toregge/add-memory-usage-to-attribute-header-tags

Add memory usage to attribute header tags.

Author: toregge

searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp

@@ -22,6 +22,7 @@ const ValueType tensor_type = ValueType::from_spec("tensor<float>(x[4])");
 constexpr uint32_t num_docs = 23;
 constexpr uint64_t unique_value_count = 11;
 constexpr uint64_t total_value_count = 13;
+constexpr uint64_t memory_usage = 420042;
 constexpr uint64_t create_serial_num = 17;
 constexpr uint32_t version = 19;
 
@@ -38,6 +39,7 @@ populate_header(const HnswIPO& hnsw_params)
                            num_docs,
                            unique_value_count,
                            total_value_count,
+                           memory_usage,
                            create_serial_num,
                            version);
 
@@ -59,6 +61,7 @@ verify_roundtrip_serialization(const HnswIPO& hnsw_params_in)
     EXPECT_EQ(create_serial_num, attr_header.getCreateSerialNum());
     EXPECT_EQ(total_value_count, attr_header.get_total_value_count());
     EXPECT_EQ(unique_value_count, attr_header.get_unique_value_count());
+    EXPECT_EQ(memory_usage, attr_header.get_memory_usage());
     EXPECT_EQ(version, attr_header.getVersion());
     EXPECT_EQ(false, attr_header.getPredicateParamsSet());
     const auto& hnsw_params_out = attr_header.get_hnsw_index_params();

searchlib/src/vespa/searchlib/attribute/attribute_header.cpp

@@ -32,6 +32,7 @@ const std::string doc_id_limit_tag = "docIdLimit";
 const std::string enumerated_tag = "enumerated";
 const std::string unique_value_count_tag = "uniqueValueCount";
 const std::string total_value_count_tag = "totalValueCount";
+const std::string memory_usage_tag = "memory_usage";
 
 }
 
@@ -53,6 +54,7 @@ AttributeHeader::AttributeHeader(std::string fileName)
       _numDocs(0),
       _uniqueValueCount(0),
       _totalValueCount(0),
+      _memory_usage(0),
       _createSerialNum(0u),
       _version(0),
       _extra_tags()
@@ -69,6 +71,7 @@ AttributeHeader::AttributeHeader(std::string fileName,
                                  uint32_t numDocs,
                                  uint64_t uniqueValueCount,
                                  uint64_t totalValueCount,
+                                 uint64_t memory_usage,
                                  uint64_t createSerialNum,
                                  uint32_t version)
     : _fileName(std::move(fileName)),
@@ -83,6 +86,7 @@ AttributeHeader::AttributeHeader(std::string fileName,
       _numDocs(numDocs),
       _uniqueValueCount(uniqueValueCount),
       _totalValueCount(totalValueCount),
+      _memory_usage(memory_usage),
       _createSerialNum(createSerialNum),
       _version(version),
       _flush_duration(std::chrono::steady_clock::duration::zero())
@@ -151,6 +155,9 @@ AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header)
     if (header.hasTag(unique_value_count_tag)) {
         _uniqueValueCount = header.getTag(unique_value_count_tag).asInteger();
     }
+    if (header.hasTag(memory_usage_tag)) {
+        _memory_usage = header.getTag(memory_usage_tag).asInteger();
+    }
     if (header.hasTag(versionTag)) {
         _version = header.getTag(versionTag).asInteger();
     }
@@ -177,6 +184,7 @@ AttributeHeader::addTags(vespalib::GenericHeader &header) const
     }
     header.putTag(Tag(unique_value_count_tag, _uniqueValueCount));
     header.putTag(Tag(total_value_count_tag, _totalValueCount));
+    header.putTag(Tag(memory_usage_tag, _memory_usage));
     header.putTag(Tag(doc_id_limit_tag, _numDocs));
     header.putTag(Tag("frozen", 0));
     header.putTag(Tag("fileBitSize", 0));

searchlib/src/vespa/searchlib/attribute/attribute_header.h

@@ -32,6 +32,7 @@ class AttributeHeader {
     uint32_t    _numDocs;
     uint64_t    _uniqueValueCount;
     uint64_t    _totalValueCount;
+    uint64_t    _memory_usage;
     uint64_t    _createSerialNum;
     uint32_t    _version;
     std::chrono::steady_clock::duration _flush_duration;
@@ -51,6 +52,7 @@ class AttributeHeader {
                     uint32_t numDocs,
                     uint64_t uniqueValueCount,
                     uint64_t totalValueCount,
+                    uint64_t memory_usage,
                     uint64_t createSerialNum,
                     uint32_t version);
     ~AttributeHeader();
@@ -76,6 +78,7 @@ class AttributeHeader {
     void addTags(vespalib::GenericHeader &header) const;
     vespalib::GenericHeader& get_extra_tags() noexcept { return _extra_tags; }
     std::chrono::steady_clock::duration get_flush_duration() const noexcept { return _flush_duration; }
+    uint64_t get_memory_usage() const noexcept { return _memory_usage; }
 };
 
 }

searchlib/src/vespa/searchlib/attribute/attributevector.cpp

@@ -305,6 +305,7 @@ AttributeVector::createAttributeHeader(std::string_view fileName) const {
                                       getCommittedDocIdLimit(),
                                       getUniqueValueCount(),
                                       getTotalValueCount(),
+                                      getStatus().get_used_minus_dead_and_onhold(),
                                       getCreateSerialNum(),
                                       getVersion());
 }