Store indices to disk (#279)

Author: 16bit-ykiko

include/Compiler/CompilationUnit.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <chrono>
+
 #include "Directive.h"
 #include "Compiler/Diagnostic.h"
 #include "AST/SymbolID.h"
@@ -166,6 +168,10 @@ class CompilationUnit {
 
     auto top_level_decls() -> llvm::ArrayRef<clang::Decl*>;
 
+    std::chrono::milliseconds build_at();
+
+    std::chrono::milliseconds build_duration();
+
     clang::LangOptions& lang_options();
 
     clang::ASTContext& context();

include/Index/IncludeGraph.h

@@ -12,13 +12,15 @@ namespace clice::index {
 
 struct IncludeLocation {
     /// The file path of the include directive.
-    std::uint32_t path = -1;
+    std::uint32_t path_id = -1;
 
     /// The line number of the include directive, 1-based.
     std::uint32_t line = -1;
 
     /// The include location that introduces this file.
     std::uint32_t include = -1;
+
+    friend bool operator== (const IncludeLocation&, const IncludeLocation&) = default;
 };
 
 struct IncludeGraph {
@@ -51,7 +53,7 @@ struct IncludeGraph {
     std::uint32_t path_id(clang::FileID fid) {
         auto include = include_location_id(fid);
         if(include != -1) {
-            return locations[include].path;
+            return locations[include].path_id;
         } else {
             return paths.size() - 1;
         }

include/Index/MergedIndex.h

@@ -1,156 +1,83 @@
 #pragma once
 
 #include "TUIndex.h"
-#include "Support/Bitmap.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/MemoryBuffer.h"
 
-namespace llvm {
-
-template <typename... Ts>
-unsigned dense_hash(const Ts&... ts) {
-    return llvm::DenseMapInfo<std::tuple<Ts...>>::getHashValue(std::tuple{ts...});
-}
-
-template <>
-struct DenseMapInfo<clice::index::Occurrence> {
-    using R = clice::LocalSourceRange;
-    using V = clice::index::Occurrence;
-
-    inline static V getEmptyKey() {
-        return V(R(-1, 0), 0);
-    }
-
-    inline static V getTombstoneKey() {
-        return V(R(-2, 0), 0);
-    }
-
-    static auto getHashValue(const V& v) {
-        return dense_hash(v.range.begin, v.range.end, v.target);
-    }
-
-    static bool isEqual(const V& lhs, const V& rhs) {
-        return lhs.range == rhs.range && lhs.target == rhs.target;
-    }
-};
+namespace clice::index {
 
-template <>
-struct DenseMapInfo<clice::index::Relation> {
-    using R = clice::index::Relation;
+class MergedIndex {
+private:
+    struct Impl;
 
-    inline static R getEmptyKey() {
-        return R{
-            .kind = clice::RelationKind(),
-            .range = clice::LocalSourceRange(-1, 0),
-            .target_symbol = 0,
-        };
-    }
+    using Self = MergedIndex;
 
-    inline static R getTombstoneKey() {
-        return R{
-            .kind = clice::RelationKind(),
-            .range = clice::LocalSourceRange(-2, 0),
-            .target_symbol = 0,
-        };
-    }
+    MergedIndex(std::unique_ptr<llvm::MemoryBuffer> buffer, std::unique_ptr<Impl> impl);
 
-    /// Contextual doen't take part in hashing and equality.
-    static auto getHashValue(const R& relation) {
-        return dense_hash(relation.kind.value(),
-                          relation.range.begin,
-                          relation.range.end,
-                          relation.target_symbol);
-    }
+    void load_in_memory(this Self& self);
 
-    static bool isEqual(const R& lhs, const R& rhs) {
-        return lhs.kind == rhs.kind && lhs.range == rhs.range &&
-               lhs.target_symbol == rhs.target_symbol;
-    }
-};
+public:
+    MergedIndex();
 
-}  // namespace llvm
+    MergedIndex(llvm::StringRef data);
 
-namespace clice::index {
-
-/// struct CompilationContext {
-///     /// The target of this compilation.
-///     llvm::StringRef target;
-///
-///     /// The canonical compilation command.
-///     llvm::StringRef command;
-///
-///     /// A version field for verification.
-///     std::uint32_t version;
-/// };
-///
-/// struct HeaderContext : CompilationContext {
-///     /// The include location in the include graph.
-///     std::uint32_t include;
-///
-///     /// The path of the file includes this header.
-///     llvm::StringRef path;
-/// };
-
-struct HeaderContexts {
-    std::uint32_t version = 0;
-
-    struct Context {
-        std::uint32_t include;
-        std::uint32_t canonical_id;
-
-        friend bool operator== (const Context&, const Context&) = default;
-    };
-
-    /// A array of include location and its context id.
-    llvm::SmallVector<Context> includes;
-
-    friend bool operator== (const HeaderContexts&, const HeaderContexts&) = default;
-};
+    MergedIndex(const MergedIndex&) = delete;
 
-struct MergedIndex {
-    /// For each merged index, we will give it a canonical id.
-    /// The max canonical id.
-    std::uint32_t max_canonical_id = 0;
+    MergedIndex(MergedIndex&& other);
 
-    /// We use the value of SHA256 to judge whether two indices are same.
-    /// Index with same content will be given same canonical id.
-    llvm::StringMap<std::uint32_t> canonical_cache;
+    MergedIndex& operator= (const MergedIndex&) = delete;
 
-    /// The reference count of each canonical id.
-    std::vector<std::uint32_t> canonical_ref_counts;
+    MergedIndex& operator= (MergedIndex&& other);
 
-    /// The canonical id set of removed index.
-    roaring::Roaring removed;
+    ~MergedIndex();
 
-    /// A map between source file path and its header contexts.
-    llvm::StringMap<HeaderContexts> contexts;
+    /// Load merged index from disk
+    static MergedIndex load(llvm::StringRef path);
 
-    /// All merged symbol occurrences.
-    llvm::DenseMap<Occurrence, roaring::Roaring> occurrences;
+    /// Serialize it to binary format.
+    void serialize(this const Self& self, llvm::raw_ostream& out);
 
-    /// All merged symbol relations.
-    llvm::DenseMap<SymbolHash, llvm::DenseMap<Relation, roaring::Roaring>> relations;
+    /// Lookup the occurrence in corresponding offset.
+    void lookup(this const Self& self,
+                std::uint32_t offset,
+                llvm::function_ref<bool(const Occurrence&)> callback);
 
-    /// FIXME: The content of this file.
-    /// std::string content;
+    /// Lookup the relations of given symbol.
+    void lookup(this const Self& self,
+                SymbolHash symbol,
+                RelationKind kind,
+                llvm::function_ref<bool(const Relation&)> callback);
 
-    /// Sorted occurrences cache for fast lookup.
-    std::vector<Occurrence> cache_occurrences;
+    /// Whether this index needs rebuilding.
+    bool need_update(this const Self& self, llvm::ArrayRef<llvm::StringRef> path_mapping);
 
-    void remove(llvm::StringRef path);
+    bool need_rewrite() {
+        return impl != nullptr;
+    }
 
-    void merge(llvm::StringRef path, std::uint32_t include, FileIndex& index);
+    /// Remove the index of specific path id.
+    void remove(this Self& self, std::uint32_t path_id);
 
-    std::vector<Occurrence> lookup(std::uint32_t offset);
+    /// Merge the index with given compilation context.
+    void merge(this Self& self,
+               std::uint32_t path_id,
+               std::chrono::milliseconds build_at,
+               std::vector<IncludeLocation> include_locations,
+               FileIndex& index);
 
-    void serialize(this MergedIndex& self, llvm::raw_ostream& out);
+    /// Merge the index with given header context.
+    void merge(this Self& self, std::uint32_t path_id, std::uint32_t include_id, FileIndex& index);
 
-    friend bool operator== (const MergedIndex&, const MergedIndex&) = default;
-};
+    friend bool operator== (MergedIndex& lhs, MergedIndex& rhs);
 
-struct MergedIndexView {
-    const void* data;
+private:
+    /// The binary serialization data of index. If you load merged index
+    /// from disk, we use directly access the data without deserialization
+    /// unless you want to modify it.
+    std::unique_ptr<llvm::MemoryBuffer> buffer;
 
-    MergedIndex deserialize();
+    /// The in memory data of the index.
+    std::unique_ptr<Impl> impl;
 };
 
 }  // namespace clice::index

include/Index/ProjectIndex.h

@@ -47,7 +47,7 @@ struct ProjectIndex {
 
     SymbolTable symbols;
 
-    void merge(this ProjectIndex& self, TUIndex& index);
+    llvm::SmallVector<std::uint32_t> merge(this ProjectIndex& self, TUIndex& index);
 
     void serialize(this ProjectIndex& self, llvm::raw_ostream& os);
 

include/Index/TUIndex.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <chrono>
 #include "IncludeGraph.h"
 #include "AST/SourceCode.h"
 #include "AST/SymbolKind.h"
@@ -43,6 +44,8 @@ struct FileIndex {
     llvm::DenseMap<SymbolHash, std::vector<Relation>> relations;
 
     std::vector<Occurrence> occurrences;
+
+    std::array<std::uint8_t, 32> hash();
 };
 
 struct Symbol {
@@ -59,12 +62,18 @@ struct Symbol {
 using SymbolTable = llvm::DenseMap<SymbolHash, Symbol>;
 
 struct TUIndex {
+    /// The building timestamp of this file.
+    std::chrono::milliseconds built_at;
+
+    /// The include information of this file.
     IncludeGraph graph;
 
     SymbolTable symbols;
 
     llvm::DenseMap<clang::FileID, FileIndex> file_indices;
 
+    FileIndex main_file_index;
+
     static TUIndex build(CompilationUnit& unit);
 };
 

include/Index/schema.fbs

@@ -22,19 +22,29 @@ table CacheEntry {
     canonical_id: uint;
 }
 
-struct Context {
-    include_: uint;
+struct IncludeContext {
+    include_id: uint;
     canonical_id: uint;
 }
 
-table HeaderContexts {
+table HeaderContextEntry {
+    path_id: uint;
     version: uint;
-    includes: [Context];
+    includes: [IncludeContext];
 }
 
-table HeaderContextsEntry {
-    path: string;
-    contexts: HeaderContexts;
+struct IncludeLocation {
+    path_id: uint;
+    line: uint;
+    include_id: uint;
+}
+
+table CompilationContextEntry {
+    path_id: uint;
+    version: uint;
+    canonical_id: uint;
+    build_at: ulong;
+    include_locations: [IncludeLocation];
 }
 
 table OccurrenceEntry {
@@ -67,7 +77,9 @@ table MergedIndex {
 
     canonical_cache: [CacheEntry];
 
-    contexts: [HeaderContextsEntry];
+    header_contexts: [HeaderContextEntry];
+
+    compilation_contexts: [CompilationContextEntry];
 
     occurrences: [OccurrenceEntry];