feat: add master stderr logger, simplify build flow, wait for AST in workers

- Add stderr_logger for master server in pipe/socket modes so logs are visible
- Remove scan_file, compile_graph deps check, and background indexer from build
  flow (not needed until PCH/PCM support is added)
- Add ast_ready event to stateful worker so feature handlers wait for
  compilation instead of returning null immediately
- Remove dirty document early-return in feature handlers (stale results > null)
- Rewrite integration tests with pygls and stronger assertions for features
- Format all changed files

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: 16bit-ykiko

pixi.lock

@@ -49,6 +49,8 @@ scripts = ["scripts/activate_asan.bat"]
 [feature.test.pypi-dependencies]
 pytest = "*"
 pytest-asyncio = ">=1.1.0"
+pygls = ">=2.0.0"
+lsprotocol = ">=2024.0.0"
 
 [feature.package.dependencies]
 xz = ">=5.8.1,<6"

pixi.toml

@@ -78,6 +78,11 @@ int main(int argc, const char** argv) {
         return 1;
     }
 
+    std::string self_path = llvm::sys::fs::getMainExecutable(argv[0], (void*)main);
+    if(!clice::fs::init_resource_dir(self_path)) {
+        LOG_ERROR("Cannot find the resource dir: {}", self_path);
+    }
+
     auto& mode = *opts.mode;
 
     if(mode == "stateless-worker") {
@@ -90,6 +95,8 @@ int main(int argc, const char** argv) {
     }
 
     if(mode == "pipe") {
+        clice::logging::stderr_logger("master", clice::logging::options);
+
         namespace et = eventide;
         et::event_loop loop;
 
@@ -99,8 +106,6 @@ int main(int argc, const char** argv) {
             return 1;
         }
 
-        std::string self_path = llvm::sys::fs::getMainExecutable(argv[0], (void*)main);
-
         et::ipc::JsonPeer peer(loop, std::move(*transport));
         clice::MasterServer server(loop, peer, std::move(self_path));
         server.register_handlers();
@@ -110,6 +115,8 @@ int main(int argc, const char** argv) {
     }
 
     if(mode == "socket") {
+        clice::logging::stderr_logger("master", clice::logging::options);
+
         namespace et = eventide;
         et::event_loop loop;
 
@@ -124,8 +131,6 @@ int main(int argc, const char** argv) {
 
         LOG_INFO("Listening on {}:{} ...", host, port);
 
-        std::string self_path = llvm::sys::fs::getMainExecutable(argv[0], (void*)main);
-
         auto task = [&]() -> et::task<> {
             auto client = co_await acceptor->accept();
             if(!client.has_value()) {
@@ -142,6 +147,7 @@ int main(int argc, const char** argv) {
             server.register_handlers();
 
             co_await peer.run();
+            peer.close();
             loop.stop();
         };
 

src/clice.cc

@@ -44,13 +44,17 @@ std::unique_ptr<clang::CompilerInvocation>
 
     std::unique_ptr<clang::CompilerInvocation> invocation;
 
-    /// Arguments from compilation database are already cc1
-    if(params.arguments_from_database) {
+    /// If the second argument is "-cc1", the arguments are already expanded
+    /// (e.g. from compilation database + query_toolchain). Skip driver and "-cc1"
+    /// and create invocation directly from the cc1 args.
+    bool is_cc1 = params.arguments.size() >= 2 && llvm::StringRef(params.arguments[1]) == "-cc1";
+    if(is_cc1) {
         invocation = std::make_unique<clang::CompilerInvocation>();
-        if(!clang::CompilerInvocation::CreateFromArgs(*invocation,
-                                                      llvm::ArrayRef(params.arguments).drop_front(),
-                                                      *diagnostic_engine,
-                                                      params.arguments[0])) {
+        if(!clang::CompilerInvocation::CreateFromArgs(
+               *invocation,
+               llvm::ArrayRef(params.arguments).drop_front(2),
+               *diagnostic_engine,
+               params.arguments[0])) {
             LOG_ERROR_RET(nullptr,
                           " Fail to create invocation, arguments list is: {}",
                           print_argv(params.arguments));

src/compile/compilation.cpp

@@ -75,8 +75,6 @@ struct CompilationParams {
 
     std::string directory;
 
-    bool arguments_from_database = false;
-
     /// Responsible for storing the arguments.
     std::vector<const char*> arguments;
 

src/compile/compilation.h

@@ -382,6 +382,7 @@ std::vector<const char*> query_toolchain(const QueryParams& params) {
             query_driver(params_copy.arguments,
                          [&](const char* driver, llvm::ArrayRef<const char*> cc1_args) {
                              result.emplace_back(params.callback(driver));
+                             result.emplace_back(params.callback("-cc1"));
                              for(auto arg: cc1_args) {
                                  result.emplace_back(params.callback(arg));
                              }
@@ -435,6 +436,7 @@ std::vector<const char*> query_gcc_toolchain(const QueryParams& params) {
     std::vector<const char*> result;
     query_driver(query_arguments, [&](const char* driver, llvm::ArrayRef<const char*> cc1_args) {
         result.emplace_back(params.callback(driver));
+        result.emplace_back(params.callback("-cc1"));
         for(auto arg: cc1_args) {
             result.emplace_back(params.callback(arg));
         }

src/compile/toolchain.cpp

@@ -8,8 +8,10 @@
 
 #include "eventide/ipc/lsp/position.h"
 #include "eventide/ipc/lsp/uri.h"
+#include "eventide/reflection/enum.h"
 #include "eventide/serde/json/json.h"
 #include "eventide/serde/serde/raw_value.h"
+#include "semantic/symbol_kind.h"
 #include "server/protocol.h"
 #include "support/filesystem.h"
 #include "support/logging.h"
@@ -100,32 +102,18 @@ et::task<> MasterServer::run_build_drain(std::uint32_t path_id, std::string uri)
         doc_it->second.build_requested = false;
         auto gen = doc_it->second.generation;
 
-        // Ensure PCM/PCH dependencies are ready
-        auto deps_ok = co_await compile_graph.compile_deps(path_id, loop);
-
-        // Re-lookup after co_await (map may have changed)
-        doc_it = documents.find(path_id);
-        if(doc_it == documents.end())
-            co_return;
-
-        if(!deps_ok) {
-            LOG_WARN("Dependency compilation failed for {}, skipping build", uri);
-            clear_diagnostics(uri);
-            if(!doc_it->second.build_requested) {
-                doc_it->second.build_running = false;
-                doc_it->second.drain_scheduled = false;
-                co_return;
-            }
-            continue;
-        }
-
         // Send compile request to stateful worker
         worker::CompileParams params;
         params.path = std::string(path_pool.resolve(path_id));
         params.version = doc_it->second.version;
         params.text = doc_it->second.text;
         fill_compile_args(path_pool.resolve(path_id), params.directory, params.arguments);
 
+        LOG_DEBUG("Sending compile: path={}, args={}, gen={}",
+                  params.path,
+                  params.arguments.size(),
+                  gen);
+
         auto result = co_await pool.send_stateful<worker::CompileParams>(path_id, params);
 
         // Re-lookup document (may have been closed during compile)
@@ -139,6 +127,11 @@ et::task<> MasterServer::run_build_drain(std::uint32_t path_id, std::string uri)
             // Only publish diagnostics if the generation hasn't changed
             if(doc2.generation == gen) {
                 publish_diagnostics(uri, doc2.version, result.value().diagnostics);
+            } else {
+                LOG_DEBUG("Generation mismatch ({} vs {}), dropping diagnostics for {}",
+                          doc2.generation,
+                          gen,
+                          uri);
             }
         } else {
             LOG_WARN("Compile failed for {}: {}", uri, result.error().message);
@@ -201,19 +194,6 @@ et::task<> MasterServer::load_workspace() {
 
     auto updates = cdb.load_compile_database(cdb_path);
     LOG_INFO("Loaded CDB from {} with {} entries", cdb_path, updates.size());
-
-    // Scan all source files from CDB to build include graph
-    auto all_files = cdb.files();
-    for(auto* file: all_files) {
-        auto path_id = path_pool.intern(file);
-        scan_file(path_id, file);
-    }
-
-    LOG_INFO("Initial scan complete, {} files in include graph", include_forward.size());
-
-    // Populate the index queue with all CDB files for background indexing
-    populate_index_queue();
-    LOG_INFO("Index queue populated with {} files", index_queue.size());
 }
 
 void MasterServer::scan_file(std::uint32_t path_id, llvm::StringRef path) {
@@ -223,7 +203,6 @@ void MasterServer::scan_file(std::uint32_t path_id, llvm::StringRef path) {
 
     auto results = scan_fuzzy(ctx.arguments,
                               ctx.directory,
-                              /*arguments_from_database=*/true,
                               /*content=*/{},
                               &scan_cache);
 
@@ -254,7 +233,7 @@ void MasterServer::scan_file(std::uint32_t path_id, llvm::StringRef path) {
 void MasterServer::fill_compile_args(llvm::StringRef path,
                                      std::string& directory,
                                      std::vector<std::string>& arguments) {
-    auto ctx = cdb.lookup(path);
+    auto ctx = cdb.lookup(path, {.resource_dir = true, .query_toolchain = true});
     directory = ctx.directory.str();
     arguments.clear();
     for(auto* arg: ctx.arguments) {
@@ -414,7 +393,32 @@ void MasterServer::register_handlers() {
 
         // Semantic tokens
         protocol::SemanticTokensOptions sem_opts;
-        sem_opts.legend = protocol::SemanticTokensLegend{{}, {}};
+        {
+            auto lower_first = [](std::string_view name) -> std::string {
+                std::string s(name);
+                if(!s.empty()) {
+                    s[0] = static_cast<char>(std::tolower(static_cast<unsigned char>(s[0])));
+                }
+                return s;
+            };
+
+            std::vector<std::string> token_types;
+            using SymbolKindRefl = eventide::refl::reflection<SymbolKind::Kind>;
+            for(std::size_t i = 0; i < SymbolKindRefl::member_count; ++i) {
+                token_types.push_back(lower_first(SymbolKindRefl::member_names[i]));
+            }
+
+            std::vector<std::string> token_modifiers;
+            using SymbolModRefl = eventide::refl::reflection<SymbolModifiers::Kind>;
+            for(std::size_t i = 0; i < SymbolModRefl::member_count; ++i) {
+                token_modifiers.push_back(lower_first(SymbolModRefl::member_names[i]));
+            }
+
+            sem_opts.legend = protocol::SemanticTokensLegend{
+                std::move(token_types),
+                std::move(token_modifiers),
+            };
+        }
         sem_opts.full =
             protocol::variant<protocol::boolean, protocol::SemanticTokensFullDelta>{true};
         result.capabilities.semantic_tokens_provider = std::move(sem_opts);
@@ -452,13 +456,8 @@ void MasterServer::register_handlers() {
 
         lifecycle = ServerLifecycle::Ready;
 
-        // Load CDB and build include graph in background
+        // Load CDB in background
         loop.schedule(load_workspace());
-
-        // Start background indexer coroutine if enabled
-        if(config.enable_indexing) {
-            loop.schedule(run_background_indexer());
-        }
     });
 
     // === shutdown ===
@@ -502,19 +501,6 @@ void MasterServer::register_handlers() {
         // Reset idle timer on user activity
         reset_idle_timer();
 
-        // Scan includes and register compile unit
-        scan_file(path_id, path);
-
-        // Get dependency path_ids from the forward graph
-        llvm::SmallVector<std::uint32_t> deps;
-        auto fwd_it = include_forward.find(path_id);
-        if(fwd_it != include_forward.end()) {
-            for(auto dep_id: fwd_it->second) {
-                deps.push_back(dep_id);
-            }
-        }
-        compile_graph.register_unit(path_id, deps);
-
         schedule_build(path_id, td.uri);
     });
 
@@ -570,10 +556,6 @@ void MasterServer::register_handlers() {
         // Reset idle timer on user activity
         reset_idle_timer();
 
-        // Rescan the file to update include graph
-        scan_file(path_id, path);
-
-        compile_graph.update(path_id);
         schedule_build(path_id, params.text_document.uri);
     });