diff --git a/cmake/package.cmake b/cmake/package.cmake index 769c4550c..68d99cc72 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -66,16 +66,20 @@ set(FLATBUFFERS_BUILD_GRPC OFF CACHE BOOL "" FORCE) set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "" FORCE) -# cpptrace -FetchContent_Declare( - cpptrace - GIT_REPOSITORY https://github.com/jeremy-rifkin/cpptrace.git - GIT_TAG v1.0.4 - GIT_SHALLOW TRUE -) -set(CPPTRACE_DISABLE_CXX_20_MODULES ON CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(libuv spdlog tomlplusplus croaring flatbuffers) -FetchContent_MakeAvailable(libuv spdlog tomlplusplus croaring flatbuffers cpptrace) +if(CLICE_ENABLE_TEST) + # cpptrace + FetchContent_Declare( + cpptrace + GIT_REPOSITORY https://github.com/jeremy-rifkin/cpptrace.git + GIT_TAG v1.0.4 + GIT_SHALLOW TRUE + ) + set(CPPTRACE_DISABLE_CXX_20_MODULES ON CACHE BOOL "" FORCE) + + FetchContent_MakeAvailable(cpptrace) +endif() if(WIN32) target_compile_definitions(uv_a PRIVATE _CRT_SECURE_NO_WARNINGS) diff --git a/docs/en/dev/server-plugin.md b/docs/en/dev/server-plugin.md new file mode 100644 index 000000000..cff04af39 --- /dev/null +++ b/docs/en/dev/server-plugin.md @@ -0,0 +1,74 @@ +You can implement a clice server plugin to extend clice's functionality. + +## Use case + +When you use `clice` as LSP backend of LLM agents, e.g. claude code, you can add plugin to provide some extra features. + +## Writing a plugin + +When a plugin is loaded by the server, it will call `clice_get_server_plugin_info` to obtain information about this plugin and about how to register its customization points. + +This function needs to be implemented by the plugin, see the example below: + +```c++ +extern "C" ::clice::PluginInfo LLVM_ATTRIBUTE_WEAK +clice_get_server_plugin_info() { + return { + CLICE_PLUGIN_API_VERSION, "MyPlugin", "v0.1", CLICE_PLUGIN_DEF_HASH, + [](ServerPluginBuilder builder) { ... } + }; +} +``` + +See [PluginDef.h](/include/Server/PluginDef.h) for more details. + +## Compiling a plugin + +The plugin must be compiled with the same dependencies and compiler options as clice, otherwise it will cause undefined behavior. [config/llvm-manifest.json](/config/llvm-manifest.json) defines the build information used by clice. + +## Loading plugins + +For security reasons, clice does not allow loading plugins through configuration files, but must specify the plugin path through command line options. + +When `clice` starts, it will load all plugins specified in the command line. You can specify the plugin path through the `--plugin-path` option. + +```shell +$ clice --plugin-path /path/to/my-plugin.so +``` + +## Getting content of `CLICE_PLUGIN_DEF_HASH` + +There are two values to return in the `clice_get_server_plugin_info` function. + +- `CLICE_PLUGIN_API_VERSION` is used to ensure compability of the `clice_get_server_plugin_info` function between the plugin and the server. +- `CLICE_PLUGIN_DEF_HASH` is used to ensure the consistency of the C++ declarations between the plugin and the server. + +To debug the content of `CLICE_PLUGIN_DEF_HASH`, you can run following command: + +```shell +$ git clone https://github.com/clice-io/clice.git +$ cd clice +$ git checkout `clice --version --git-describe` +$ python scripts/plugin-def.py content +``` + +You will get a C source code file, content of which is like this: + +```cpp +#if 0 +// begin of config/llvm-manifest.json +[ + { + "version": "21.1.4+r1", + "filename": "arm64-macos-clang-debug-asan.tar.xz", + "sha256": "7da4b7d63edefecaf11773e7e701c575140d1a07329bbbb038673b6ee4516ff5", + "lto": false, + "asan": true, + "platform": "macosx", + "build_type": "Debug" + }, + ... +] +... +#endif +``` diff --git a/docs/zh/dev/server-plugin.md b/docs/zh/dev/server-plugin.md new file mode 100644 index 000000000..429f000dd --- /dev/null +++ b/docs/zh/dev/server-plugin.md @@ -0,0 +1,74 @@ +你可以在 clice 中实现一个 server plugin 来扩展 clice 的功能。 + +## 用例 + +当你使用 `clice` 作为 LLM 代理的 LSP 后端时，比如 claude code，你可以添加插件来提供一些额外功能。 + +## 编写插件 + +当一个插件被服务器加载时，它会调用 `clice_get_server_plugin_info` 来获取关于这个插件的信息以及如何注册它的定制点。 + +这个函数需要由插件实现，请参考下面的示例： + +```cpp +extern "C" ::clice::PluginInfo LLVM_ATTRIBUTE_WEAK +clice_get_server_plugin_info() { + return { + CLICE_PLUGIN_API_VERSION, "MyPlugin", "v0.1", CLICE_PLUGIN_DEF_HASH, + [](ServerPluginBuilder builder) { ... } + }; +} +``` + +请参考 [PluginDef.h](/include/Server/PluginDef.h) 了解更多细节。 + +## 编译插件 + +插件必须使用与 clice 一致的依赖和编译器选项来编译，否则会导致 undefined behavior。[config/llvm-manifest.json](/config/llvm-manifest.json) 中定义了 clice 使用的构建信息。 + +## 加载插件 + +为了安全考虑，clice 不允许通过配置文件来加载插件，而必须通过命令行选项来指定插件的路径。 + +在 `clice` 启动时，它会加载所有在命令行中指定的插件。你可以通过 `--plugin-path` 选项来指定插件的路径。 + +```shell +$ clice --plugin-path /path/to/my-plugin.so +``` + +## 获取 `CLICE_PLUGIN_DEF_HASH` 的内容 + +在 `clice_get_server_plugin_info` 函数中需要返回两个值。 + +- `CLICE_PLUGIN_API_VERSION` 用于确保插件和服务器之间的 `clice_get_server_plugin_info` 函数的一致性。 +- `CLICE_PLUGIN_DEF_HASH` 用于确保插件和服务器之间的 C++ 声明的一致性。 + +要调试 `CLICE_PLUGIN_DEF_HASH` 的内容，你可以运行以下命令： + +```shell +$ git clone https://github.com/clice-io/clice.git +$ cd clice +$ git checkout `clice --version --git-describe` +$ python scripts/plugin-def.py content > /tmp/plugin-proto.h +``` + +你将会得到一个 C 源码格式的文件，内容大致如下： + +```cpp +#if 0 +// begin of config/llvm-manifest.json +[ + { + "version": "21.1.4+r1", + "filename": "arm64-macos-clang-debug-asan.tar.xz", + "sha256": "7da4b7d63edefecaf11773e7e701c575140d1a07329bbbb038673b6ee4516ff5", + "lto": false, + "asan": true, + "platform": "macosx", + "build_type": "Debug" + }, + ... +] +... +#endif +``` diff --git a/include/Protocol/Basic.h b/include/Protocol/Basic.h index c0ee8ffa8..801a814a0 100644 --- a/include/Protocol/Basic.h +++ b/include/Protocol/Basic.h @@ -6,6 +6,8 @@ #include #include +#include "llvm/Support/JSON.h" + namespace clice::proto { using integer = std::int32_t; @@ -15,6 +17,8 @@ using decimal = double; using string = std::string; +using any = llvm::json::Value; + template using array = std::vector; diff --git a/include/Protocol/Feature/ExecuteCommand.h b/include/Protocol/Feature/ExecuteCommand.h new file mode 100644 index 000000000..6b0333f02 --- /dev/null +++ b/include/Protocol/Feature/ExecuteCommand.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../Basic.h" + +namespace clice::proto { + +struct ExecuteCommandParams { + string command; + array arguments; +}; + +struct TextDocumentParams { + /// The text document. + TextDocumentIdentifier textDocument; +}; + +} // namespace clice::proto diff --git a/include/Protocol/TextDocument.h b/include/Protocol/TextDocument.h index b06621b55..a2f4e31af 100644 --- a/include/Protocol/TextDocument.h +++ b/include/Protocol/TextDocument.h @@ -11,6 +11,7 @@ #include "Feature/DocumentHighlight.h" #include "Feature/DocumentLink.h" #include "Feature/DocumentSymbol.h" +#include "Feature/ExecuteCommand.h" #include "Feature/FoldingRange.h" #include "Feature/Formatting.h" #include "Feature/Hover.h" diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h index eb1b591b4..2884ee7a3 100644 --- a/include/Server/Indexer.h +++ b/include/Server/Indexer.h @@ -44,6 +44,11 @@ class Indexer { if(it2 != project_index.indices.end()) { auto path = project_index.path_pool.path(it2->second); it->second = index::MergedIndex::load(path); + } else { + std::println(stderr, + "failed to load project index for path_id: {} {}", + path_id, + project_index.indices.size()); } return it->second; @@ -67,6 +72,14 @@ class Indexer { /// TODO: Types ... + bool empty() const { + return project_index.indices.empty(); + } + + size_t size() const { + return project_index.indices.size(); + } + private: CompilationDatabase& database; diff --git a/include/Server/Plugin.h b/include/Server/Plugin.h new file mode 100644 index 000000000..f60a66859 --- /dev/null +++ b/include/Server/Plugin.h @@ -0,0 +1,59 @@ +#pragma once +#include + +#include "PluginProtocol.h" + +#include "llvm/ADT/StringRef.h" + +// clang-format off +/// Run `python scripts/plugin-def.py update` to update the hash. +#define CLICE_PLUGIN_DEF_HASH "sha256:e7f911c923f9cdcec6c0edea328292bd48771f7139d4489ed694a72e9af33fc1" +// clang-format on + +namespace clice { + +/// The hash of the definitions exposed to server plugins. +constexpr std::string_view plugin_definition_hash = CLICE_PLUGIN_DEF_HASH; + +class Server; + +struct ServerPluginBuilder; + +/// A loaded server plugin. +/// +/// An instance of this class wraps a loaded server plugin and gives access to its interface. +class Plugin { +public: + /// Attempts to load a server plugin from a given file. + /// + /// Returns an error if either the library cannot be found or loaded, + /// there is no public entry point, or the plugin implements the wrong API + /// version. + static std::expected load(const std::string& file_path); + + /// Gets the file path of the loaded plugin. + llvm::StringRef file_path() const; + + /// Gets the name of the loaded plugin. + llvm::StringRef name() const; + + /// Gets the version of the loaded plugin. + llvm::StringRef version() const; + + /// Registers the server callbacks for the loaded plugin. + void register_server_callbacks(ServerPluginBuilder& builder) const; + +public: + struct Self; + + Plugin(Self* self) : self(self) {} + + Self* operator->() { + return self; + } + +protected: + Self* self; +}; + +} // namespace clice diff --git a/include/Server/PluginProtocol.h b/include/Server/PluginProtocol.h new file mode 100644 index 000000000..883bd73a2 --- /dev/null +++ b/include/Server/PluginProtocol.h @@ -0,0 +1,112 @@ +#pragma once + +/// The API version of the clice plugin. +/// Update this version when you change: +/// - The definition of struct `PluginInfo`. +/// - The definition of function `clice_get_server_plugin_info`. +/// Note: you don't have to update this version if you only change other APIs, which is guaranteed +/// by the `PluginInfo::definition_hash`. +#define CLICE_PLUGIN_API_VERSION 1 + +#include + +#include "Async/Async.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/JSON.h" + +namespace clice { + +class Server; + +struct ServerPluginBuilder; +/// Defines the library APIs that loads a plugin. +extern "C" { + /// A C-compatible struct that contains information about the plugin. + struct PluginInfo { + /// The clice API version of the plugin. + uint32_t api_version; + /// The name of the plugin. + const char* name; + /// The version of the plugin. + const char* version; + /// The plugin definition hash. + const char* definition_hash; + /// Registers the server callbacks for the loaded plugin. + void (*register_server_callbacks)(ServerPluginBuilder& builder); + }; + + /// The public entry point for a server plugin. + /// + /// When a plugin is loaded by the server, it will call this entry point to + /// obtain information about this plugin and about how to register its customization points. + /// This function needs to be implemented by the plugin, see the example below: + /// + /// ``` + /// extern "C" ::clice::PluginInfo LLVM_ATTRIBUTE_WEAK + /// clice_get_server_plugin_info() { + /// return { + /// CLICE_PLUGIN_API_VERSION, "MyPlugin", "v0.1", CLICE_PLUGIN_DEF_HASH, + /// [](ServerPluginBuilder builder) { ... } + /// }; + /// } + /// ``` + PluginInfo LLVM_ATTRIBUTE_WEAK clice_get_server_plugin_info(); +} + +struct ServerRef { +public: + struct Self; + + ServerRef(Self* self) : self(self) {} + + Self* operator->() const { + return self; + } + + Server& server() const; + +protected: + Self* self; +}; + +/// Defines the library APIs to register callbacks for a plugin. +struct ServerPluginBuilder { +public: + ServerPluginBuilder(ServerRef server_ref) : server_ref(server_ref) {} + + /// Gets a reference to the server. + auto get_server_ref() const -> ServerRef { + return server_ref; + } + +#define CliceServerPluginAPI(METHOD, ...) void METHOD(void* plugin_data, __VA_ARGS__) + + using lifecycle_hook_t = async::Task<> (*)(ServerRef server, void* plugin_data); + + /// Registers a callback to be called when the server is initialized. + CliceServerPluginAPI(on_initialize, lifecycle_hook_t callback); + /// Registers a callback to be called when the server is initialized. + CliceServerPluginAPI(on_initialized, lifecycle_hook_t callback); + /// Registers a callback to be called when the server is shutdown. + CliceServerPluginAPI(on_shutdown, lifecycle_hook_t callback); + /// Registers a callback to be called when the server is exiting. + CliceServerPluginAPI(on_exit, lifecycle_hook_t callback); + /// Registers a callback to be called when the server's configuration is changed. + CliceServerPluginAPI(on_did_change_configuration, lifecycle_hook_t callback); + using command_handler_t = + async::Task (*)(ServerRef server, + void* plugin_data, + llvm::ArrayRef arguments); + /// Registers a callback to be called when a command is received from the LSP client. + CliceServerPluginAPI(register_commmand_handler, + llvm::StringRef command, + command_handler_t callback); +#undef CliceServerPluginAPI + +protected: + ServerRef server_ref; +}; + +} // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index 9a2794fff..46d5072c4 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -3,6 +3,7 @@ #include "Config.h" #include "Convert.h" #include "Indexer.h" +#include "Plugin.h" #include "Async/Async.h" #include "Compiler/Command.h" #include "Compiler/Diagnostic.h" @@ -10,6 +11,8 @@ #include "Feature/DocumentLink.h" #include "Protocol/Protocol.h" +#include + namespace clice { struct OpenFile { @@ -192,6 +195,8 @@ class Server { private: using Result = async::Task; + auto on_execute_command(proto::ExecuteCommandParams params) -> Result; + auto on_completion(proto::CompletionParams params) -> Result; auto on_hover(proto::HoverParams params) -> Result; @@ -241,6 +246,20 @@ class Server { config::Config config; Indexer indexer; + +private: + friend struct ServerPluginBuilder; + using lifecycle_hook_t = llvm::unique_function()>; + using command_handler_t = llvm::unique_function( + llvm::ArrayRef arguments)>; + + std::vector initialize_hooks; + std::vector initialized_hooks; + std::vector shutdown_hooks; + std::vector exit_hooks; + std::vector did_change_configuration_hooks; + llvm::StringMap command_handlers; + std::vector plugins; }; } // namespace clice diff --git a/include/Server/Utility.h b/include/Server/Utility.h new file mode 100644 index 000000000..f4ddd06dc --- /dev/null +++ b/include/Server/Utility.h @@ -0,0 +1,3 @@ +#pragma once + +#define bail(...) std::unexpected(std::format(__VA_ARGS__)) diff --git a/scripts/plugin-def.py b/scripts/plugin-def.py new file mode 100644 index 000000000..1b5b01b0e --- /dev/null +++ b/scripts/plugin-def.py @@ -0,0 +1,98 @@ +import hashlib +import re +from pathlib import Path +import sys + +clice_apis = [] + +# all of the files in `include/`, other than `include/Server/Plugin.h` +for path in Path("include/").glob("**/*.h"): + if path.name == "Plugin.h": + continue + clice_apis.append(path) + +clice_apis = [ + # the dependencies of the clice. + Path("config/llvm-manifest.json"), + # the clice C/C++ sources. + *sorted(clice_apis), +] + + +def fatal(message: str): + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def sha256sum(paths: list[Path]) -> str: + digest = hashlib.sha256() + for path in paths: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def clice_api_content(): + content = [] + content.append("#if 0") + for path in clice_apis: + with path.open("r") as file: + content.append(f"// begin of {path}") + content.append(file.read()) + content.append(f"// end of {path}") + content.append("#endif") + return "\n".join(content) + + +def plugin_def_hash(): + hash_val = sha256sum(clice_apis) + return f"sha256:{hash_val}" + + +def update(): + hash_val = plugin_def_hash() + with open("include/Server/Plugin.h", "r") as file: + content = file.read() + content = re.sub( + r"#define CLICE_PLUGIN_DEF_HASH .*", + f'#define CLICE_PLUGIN_DEF_HASH "{hash_val}"', + content, + ) + with open("include/Server/Plugin.h", "w") as file: + file.write(content) + + +def check(): + hash_val = plugin_def_hash() + with open("include/Server/Plugin.h", "r") as file: + content = file.read() + match = re.search(r'#define CLICE_PLUGIN_DEF_HASH "(.*)"', content) + if match is None: + fatal("plugin def hash not found in include/Server/Plugin.h") + if match.group(1) != hash_val: + fatal( + f"plugin def hash mismatch in include/Server/Plugin.h, expected: {hash_val}, actual: {match.group(1)}" + ) + print(f"plugin def hash is up to date: {hash_val}") + + +def main(): + if len(sys.argv) > 1: + if sys.argv[1] == "update": + update() + check() + elif sys.argv[1] == "check": + check() + elif sys.argv[1] == "content": + print(clice_api_content()) + else: + fatal( + f"invalid command: {sys.argv[1]}, expected: update, check", + ) + else: + fatal("no command provided, expected: update, check") + + +if __name__ == "__main__": + main() diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index d9afb999c..46b1333ca 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -643,10 +643,23 @@ std::vector CompilationDatabase::load_compile_database(llvm::StringR llvm::BumpPtrAllocator local; llvm::StringSaver saver(local); llvm::SmallVector agrs; + auto hasCC1 = false; for(auto& argument: *arguments) { if(argument.kind() == json::Value::String) { agrs.emplace_back(saver.save(*argument.getAsString()).data()); + if(argument.getAsString() == "-cc1") { + hasCC1 = true; + break; + } + } + } + if(hasCC1) { + std::println(stderr, "cannot handle arguments: {}", *file); + for(auto& arg: *arguments) { + std::print(stderr, "{} ", arg); } + std::println(stderr, ""); + continue; } item.info = self->save_compilation_info(*file, *directory, agrs); } else if(command) { diff --git a/src/Compiler/Compilation.cpp b/src/Compiler/Compilation.cpp index 5c28bcf4c..6ecaee428 100644 --- a/src/Compiler/Compilation.cpp +++ b/src/Compiler/Compilation.cpp @@ -3,13 +3,13 @@ #include "Implement.h" #include "AST/Utility.h" #include "Compiler/Command.h" -#include "Compiler/Diagnostic.h" #include "Support/Logging.h" -#include "llvm/Support/Error.h" +#include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/MultiplexConsumer.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Lex/PreprocessorOptions.h" +#include "clang-tidy/ClangTidyModuleRegistry.h" namespace clice { @@ -114,7 +114,7 @@ auto create_invocation(CompilationUnitRef::Self& self, *diagnostic_engine, params.arguments[0])) { LOG_ERROR_RET(nullptr, - " Fail to create invocation, arguments list is: {}", + " Fail to create invocation from database, arguments list is: {}", print_argv(params.arguments)); } } else { diff --git a/src/Compiler/Toolchain.cpp b/src/Compiler/Toolchain.cpp index c44064cd7..19e550fc1 100644 --- a/src/Compiler/Toolchain.cpp +++ b/src/Compiler/Toolchain.cpp @@ -287,7 +287,7 @@ CompilerFamily driver_family(llvm::StringRef driver) { std::vector query_toolchain(const QueryParams& params) { auto arguments = params.arguments; - llvm::StringRef driver = arguments[0]; + llvm::StringRef driver = "clang"; // arguments[0]; /// Note: The name used to invoke the compiler driver affects its behavior. /// For example, `/usr/bin/clang++` is often a symbolic link to diff --git a/src/Server/Config.cpp b/src/Server/Config.cpp index 6247afded..3b4f8c4d3 100644 --- a/src/Server/Config.cpp +++ b/src/Server/Config.cpp @@ -96,7 +96,8 @@ static void parse_toml(Object& object, auto&& value, Config& config) { auto Config::parse(llvm::StringRef workspace) -> std::expected { this->workspace = workspace; - auto path = path::join(workspace, "clice.toml"); + // auto path = path::join(workspace, "clice.toml"); + auto path = "/mnt/sda2/tfuzz/packages/tfuzz/src/clice.toml"; std::string error_message; if(fs::exists(path)) { diff --git a/src/Server/Implement.h b/src/Server/Implement.h new file mode 100644 index 000000000..645fcb97b --- /dev/null +++ b/src/Server/Implement.h @@ -0,0 +1,19 @@ +#pragma once + +#include "Server/Plugin.h" +#include "Server/Server.h" + +namespace clice { + +struct ServerRef::Self { +public: + Self(Server* server_instance) : server_instance(server_instance) {} + + Server& server() const { + return *server_instance; + } + + Server* server_instance; +}; + +} // namespace clice diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index ad4dbd28c..8b1bf4989 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -34,6 +34,7 @@ async::Task<> Indexer::index(llvm::StringRef path) { }); if(!tu_index) { + std::println(stderr, "Fail to index for {}, because of compile errors", path); co_return; } @@ -108,9 +109,15 @@ void Indexer::load_from_disk() { if(auto content = fs::read(output_path); content && !content->empty()) { /// FIXME: from should return a expected ... project_index = index::ProjectIndex::from(content->data()); - LOG_INFO("Load project index form {} successfully", output_path); + std::println(stderr, + "Load project index form {} successfully, indices: {}", + output_path, + project_index.indices.size()); + for(auto& [path_id, index]: project_index.indices) { + std::println(stderr, "path_id: {} {}", path_id, project_index.path_pool.path(path_id)); + } } else { - LOG_INFO("Fail to load project index form {}", output_path); + std::println(stderr, "Fail to load project index form {}", output_path); } /// FIXME: check indices update .... @@ -122,8 +129,10 @@ void Indexer::save_to_disk() { return; } + auto not_saved = 0; for(auto& [path_id, index]: in_memory_indices) { if(index.need_rewrite()) { + LOG_INFO("save index: {} {}", path_id, project_index.path_pool.path(path_id)); auto path = project_index.path_pool.path(path_id); std::string output_path; @@ -147,8 +156,12 @@ void Indexer::save_to_disk() { auto opath_id = project_index.path_pool.path_id(output_path); project_index.indices.try_emplace(path_id, opath_id); LOG_INFO("Successfully save index for {} to {}", path, output_path); + } else { + LOG_INFO("not save index: {} {}", path_id, project_index.path_pool.path(path_id)); + not_saved += 1; } } + LOG_INFO("not save index: {}", not_saved); std::string output_path = path::join(config.project.index_dir, "project.idx"); @@ -167,6 +180,7 @@ auto Indexer::lookup(llvm::StringRef path, std::uint32_t offset, RelationKind ki std::vector locations; auto path_id = project_index.path_pool.path_id(path); + std::println(stderr, "path_id: {}", path_id); auto& index = get_index(path_id); llvm::SmallVector occurrences; @@ -174,6 +188,7 @@ auto Indexer::lookup(llvm::StringRef path, std::uint32_t offset, RelationKind ki occurrences.emplace_back(o); return true; }); + std::println(stderr, "occurrences: {}", occurrences.size()); if(occurrences.empty()) { co_return locations; } diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index 02dd241d0..3e16b716b 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -1,3 +1,4 @@ +#include "Server/Plugin.h" #include "Server/Server.h" #include @@ -46,6 +47,11 @@ async::Task Server::on_initialize(proto::InitializeParams params) { } } + /// Run initialize hooks. + for(auto& hook: initialize_hooks) { + co_await hook(); + } + /// Load cache info. load_cache_info(); @@ -108,18 +114,40 @@ async::Task Server::on_initialize(proto::InitializeParams params) { } async::Task<> Server::on_initialized(proto::InitializedParams) { + /// Run initialized hooks. + for(auto& hook: initialized_hooks) { + co_await hook(); + } + indexer.load_from_disk(); - co_await indexer.index_all(); + if(indexer.empty()) { + LOG_INFO("project_index is empty, index all"); + co_await indexer.index_all(); + } else { + LOG_INFO("project_index is not empty"); + } + std::println(stderr, "project_index indexed"); co_return; } async::Task Server::on_shutdown(proto::ShutdownParams params) { + /// Run shutdown hooks. + for(auto& hook: shutdown_hooks) { + co_await hook(); + } + co_return json::Value(nullptr); } async::Task<> Server::on_exit(proto::ExitParams params) { + /// Run exit hooks. + for(auto& hook: exit_hooks) { + co_await hook(); + } + save_cache_info(); indexer.save_to_disk(); + async::stop(); co_return; } diff --git a/src/Server/Plugin.cpp b/src/Server/Plugin.cpp new file mode 100644 index 000000000..daf02dbdf --- /dev/null +++ b/src/Server/Plugin.cpp @@ -0,0 +1,158 @@ +#include "Server/Plugin.h" + +#include "Implement.h" +#include "Server/Utility.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/DynamicLibrary.h" + +namespace clice { + +Server& ServerRef::server() const { + return self->server(); +} + +struct Plugin::Self { + /// The file path of the plugin. + std::string file_path; + /// The dynamic library data of the plugin. + llvm::sys::DynamicLibrary library; + /// The name of the plugin. + std::string name; + /// The version of the plugin. + std::string version; + /// Registers the server callbacks for the loaded plugin. + void (*register_server_callbacks)(ServerPluginBuilder& builder); +}; + +std::expected Plugin::load(const std::string& file_path) { + std::string err; + auto library = llvm::sys::DynamicLibrary::getPermanentLibrary(file_path.c_str(), &err); + if(!library.isValid()) { + return bail("Could not load library '{}': {}", file_path, err); + } + + Plugin P{ + /// We currently never destroy plugins, so this is not a memory leak. + new Self{file_path, library} + }; + + /// `clice_get_server_plugin_info` should be resolved to the definition from the plugin + /// we are currently loading. + intptr_t get_details_fn = (intptr_t)library.getAddressOfSymbol("clice_get_server_plugin_info"); + if(!get_details_fn) { + return bail( + "The symbol `clice_get_server_plugin_info` is not found in '{}'. Is this a clice server plugin?", + file_path); + } + + auto info = reinterpret_cast(get_details_fn)(); + + /// First, we check whether the plugin is compatible with the clice plugin API. + if(info.api_version != CLICE_PLUGIN_API_VERSION) { + return bail("Wrong API version on plugin '{}'. Got version {}. Supported version is {}.", + file_path, + info.api_version, + CLICE_PLUGIN_API_VERSION); + } + + /// Then, we safely get definition hash from the plugin, and check if it is consistent with + /// the expected hash. This ensures that the plugin has consistent declarations with the + /// server. + std::string definition_hash = info.definition_hash; + if(plugin_definition_hash.size() != definition_hash.size()) { + return bail("Wrong definition hash size on plugin '{}'. Got {}, expected {} ({}).", + file_path, + definition_hash.size(), + plugin_definition_hash.size(), + plugin_definition_hash); + } + + /// If there is any non-printable character in the definition hash, this is likely a bug in the + /// plugin. We cannot even print the `definition_hash` in this case. + if(std::ranges::any_of(definition_hash, [](char c) { return !std::isprint(c); })) { + return bail("Corrupt definition hash on plugin '{}'. This is likely a bug in the plugin.", + file_path); + } + + if(definition_hash != CLICE_PLUGIN_DEF_HASH) { + return bail("Wrong definition hash on plugin '{}'. Got '{}', expected '{}'.", + file_path, + definition_hash, + CLICE_PLUGIN_DEF_HASH); + } + + /// A plugin must implement the `register_server_callbacks` function. + if(!info.register_server_callbacks) { + return bail("Empty `register_server_callbacks` function in plugin '{}'.", file_path); + } + + P->name = info.name; + P->version = info.version; + P->register_server_callbacks = info.register_server_callbacks; + + return P; +} + +llvm::StringRef Plugin::file_path() const { + return self->file_path; +} + +llvm::StringRef Plugin::name() const { + return self->name; +} + +llvm::StringRef Plugin::version() const { + return self->version; +} + +using command_handler_t = + async::Task (*)(ServerRef server, + const llvm::ArrayRef& arguments); + +void ServerPluginBuilder::on_initialize(void* plugin_data, lifecycle_hook_t callback) { + auto server = server_ref; + server_ref.server().initialize_hooks.push_back( + [=]() -> async::Task<> { co_await callback(server, plugin_data); }); +} + +void ServerPluginBuilder::on_initialized(void* plugin_data, lifecycle_hook_t callback) { + auto server = server_ref; + server_ref.server().initialized_hooks.push_back( + [=]() -> async::Task<> { co_await callback(server, plugin_data); }); +} + +void ServerPluginBuilder::on_shutdown(void* plugin_data, lifecycle_hook_t callback) { + auto server = server_ref; + server_ref.server().shutdown_hooks.push_back( + [=]() -> async::Task<> { co_await callback(server, plugin_data); }); +} + +void ServerPluginBuilder::on_exit(void* plugin_data, lifecycle_hook_t callback) { + auto server = server_ref; + server_ref.server().exit_hooks.push_back( + [=]() -> async::Task<> { co_await callback(server, plugin_data); }); +} + +void ServerPluginBuilder::on_did_change_configuration(void* plugin_data, + lifecycle_hook_t callback) { + auto server = server_ref; + server_ref.server().did_change_configuration_hooks.push_back( + [=]() -> async::Task<> { co_await callback(server, plugin_data); }); +} + +void ServerPluginBuilder::register_commmand_handler(void* plugin_data, + llvm::StringRef command, + command_handler_t callback) { + auto server = server_ref; + auto [_, inserted] = server_ref.server().command_handlers.try_emplace( + command, + [=](llvm::ArrayRef arguments) -> async::Task { + co_return callback(server, plugin_data, arguments); + }); + if(!inserted) { + LOG_ERROR("Command handler already registered for command '{}'.", command); + } +} + +} // namespace clice diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 451ba367f..4fac05e5c 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -1,7 +1,16 @@ #include "Server/Server.h" +#include "Compiler/CompilationUnit.h" +#include "Protocol/Basic.h" +#include "Support/FileSystem.h" #include "Support/Logging.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/ASTMatchers/ASTMatchersInternal.h" +#include "clang/ASTMatchers/ASTMatchersMacros.h" +#include + namespace clice { ActiveFileManager::ActiveFile& ActiveFileManager::lru_put_impl(llvm::StringRef path, @@ -102,6 +111,8 @@ Server::Server() : indexer(database, config, kind) { register_callback<&Server::on_shutdown>("shutdown"); register_callback<&Server::on_exit>("exit"); + register_callback<&Server::on_execute_command>("workspace/executeCommand"); + register_callback<&Server::on_did_open>("textDocument/didOpen"); register_callback<&Server::on_did_change>("textDocument/didChange"); register_callback<&Server::on_did_save>("textDocument/didSave"); @@ -184,4 +195,289 @@ async::Task<> Server::on_receive(json::Value value) { co_return; } +/// Matches named declarations with a specific name. +/// +/// See \c hasName() and \c hasAnyName() in ASTMatchers.h for details. +class ContainOffsetMatcher : + public clang::ast_matchers::internal::SingleNodeMatcherInterface { +public: + explicit ContainOffsetMatcher(std::vector> offsets) : + offsets(offsets) {} + + bool matchesNode(const clang::FunctionDecl& Node) const override { + auto& ctx = Node.getASTContext(); + auto& mgr = ctx.getSourceManager(); + const auto* body = Node.getBody(); + if(!body) { + return false; + } + auto location = body->getBeginLoc(); + auto end_location = body->getEndLoc(); + auto [begin_fid, begin_offset] = mgr.getDecomposedLoc(location); + auto [end_fid, end_offset] = mgr.getDecomposedLoc(end_location); + if(begin_fid.isInvalid() || end_fid.isInvalid() || begin_fid != end_fid) { + return false; + } + + for(const auto& [expected_fid, expected_offset]: offsets) { + if(expected_fid != begin_fid) { + continue; + } + if(begin_offset <= expected_offset && expected_offset < end_offset) [[unlikely]] { + return true; + } + } + return false; + } + + clang::FileID fid; + std::vector> offsets; +}; + +inline clang::ast_matchers::internal::Matcher + containOffset(std::vector> offsets) { + return clang::ast_matchers::internal::Matcher( + new ContainOffsetMatcher(offsets)); +} + +/// A reference site is a function declaration and its content. +struct CallSite { + /// The name of the caller function of the callee. + std::string name; + /// The location of the caller function of the callee. + proto::Location location; + /// The content of the caller function of the callee. + std::string content; +}; + +/// The result of the call graph command. +struct CallGraphResult { + /// The signature of the symbol. + std::string signature; + /// The content of the symbol. + std::string content; + /// The locations of the symbol declarations. + std::vector locations; + /// The call sites of the symbol. + std::vector callSites; +}; + +async::Task Server::on_execute_command(proto::ExecuteCommandParams params) { + auto& command = params.command; + auto& arguments = params.arguments; + + if(command == "workspace/constructionInfo") { + proto::TextDocumentParams identifier = + json::deserialize(arguments[0]); + // std::string symbol = json::deserialize(arguments[1]); + auto symbol_name = *arguments[1].getAsString(); + std::println(stderr, + "constructionInfo: {} uri: {}", + symbol_name.str(), + identifier.textDocument.uri); + + auto path = mapping.to_path(identifier.textDocument.uri); + auto file = opening_files.get_or_add(path); + + { + bool hasAst = false; + { + auto guard = co_await file->ast_built_lock.try_lock(); + hasAst = !!file->ast; + } + + if(!hasAst) { + // Read the content of the file. + auto content = clice::fs::read(path); + + if(!content) { + LOG_ERROR("Failed to read the content of the file: {}", path); + co_return json::Value{}; + } + + co_await build_ast(path, *content); + } + } + + /// Try get the lock, the waiter on the lock will be resumed when + /// guard is destroyed. + auto guard = co_await file->ast_built_lock.try_lock(); + auto& ast = file->ast; + if(!ast) { + LOG_ERROR("AST not built for file: {}", path); + co_return json::Value{}; + } + + auto& ast_context = ast->context(); + + class FindDeclASTConsumer : public clang::ast_matchers::MatchFinder::MatchCallback { + public: + llvm::SmallVector matched_decls; + const clang::FunctionDecl* any_decl = nullptr; + const clang::FunctionDecl* matched_def = nullptr; + + void run(const clang::ast_matchers::MatchFinder::MatchResult& result) override { + if(auto* decl = result.Nodes.getNodeAs("func")) { + if(decl->getDefinition() == decl) { + any_decl = matched_def = decl; + } else { + if(!any_decl) { + any_decl = decl; + } + matched_decls.push_back(decl); + } + } + } + }; + + FindDeclASTConsumer consumer; + { + // match the symbol + clang::ast_matchers::MatchFinder finder; + clang::ast_matchers::DeclarationMatcher matcher = + clang::ast_matchers::functionDecl(clang::ast_matchers::hasName(symbol_name)) + .bind("func"); + finder.addMatcher(matcher, &consumer); + finder.matchAST(ast_context); + } + + // print the matched symbols + std::println(stderr, "matched decls: {}", consumer.matched_decls.size()); + std::println(stderr, "matched def: {}", !!consumer.matched_def); + + if(auto* any_decl = consumer.any_decl) { + auto location = any_decl->getLocation(); + auto [fid, offset] = ast_context.getSourceManager().getDecomposedLoc(location); + auto file_path = + ast_context.getSourceManager().getFileEntryForID(fid)->tryGetRealPathName(); + + if(file_path.empty()) { + LOG_ERROR("Failed to get the real path of the symbol in file: {} {}", + symbol_name, + path); + co_return json::Value{}; + } + + std::println(stderr, "location: {} {}", file_path, offset); + + auto locations = co_await indexer.references(file_path, offset); + + std::vector> offsets; + llvm::StringRef content = ast->interested_content(); + auto& mgr = ast_context.getSourceManager(); + PositionConverter converter(content, kind); + for(auto& location: locations) { + auto path = mapping.to_path(location.uri); + auto file_ref = mgr.getFileManager().getFileRef(path); + if(!file_ref) { + LOG_ERROR("Failed to get the file ref of the location: {} {}", + location.uri, + path); + continue; + } + auto file_id = mgr.translateFile(*file_ref); + offsets.push_back({file_id, clice::to_offset(kind, content, location.range.start)}); + } + + std::ranges::sort(offsets); + + class AllDeclASTConsumer : public clang::ast_matchers::MatchFinder::MatchCallback { + public: + llvm::SmallDenseSet matched_decls; + + void run(const clang::ast_matchers::MatchFinder::MatchResult& result) override { + if(auto* decl = result.Nodes.getNodeAs("func")) { + matched_decls.insert(decl); + } + } + }; + + AllDeclASTConsumer consumer; + { + // match the symbol + clang::ast_matchers::MatchFinder finder; + clang::ast_matchers::DeclarationMatcher matcher = + clang::ast_matchers::functionDecl(clang::ast_matchers::isDefinition(), + containOffset(offsets)) + .bind("func"); + finder.addMatcher(matcher, &consumer); + finder.matchAST(ast_context); + } + + std::vector call_sites; + std::vector matched_decls(consumer.matched_decls.begin(), + consumer.matched_decls.end()); + std::sort(matched_decls.begin(), + matched_decls.end(), + [](const clang::FunctionDecl* a, const clang::FunctionDecl* b) { + return a->getLocation() < b->getLocation(); + }); + for(auto& decl: matched_decls) { + auto location = decl->getLocation(); + auto [fid, offset] = mgr.getDecomposedLoc(location); + auto file_path = mgr.getFileEntryForID(fid)->tryGetRealPathName(); + auto content = mgr.getBufferOrNone(fid)->getBuffer(); + PositionConverter converter(content, kind); + auto begin = converter.toPosition(offset); + + // getSourceRange + auto source_range = decl->getSourceRange(); + // auto source_text = content.substr(source_range.getBegin().getOffset(), + // source_range.getEnd().getOffset()); + auto [begin_fid, begin_offset] = mgr.getDecomposedLoc(source_range.getBegin()); + auto [end_fid, end_offset] = mgr.getDecomposedLoc(source_range.getEnd()); + // todo: why +1? + auto func_content = + end_offset > begin_offset + ? content.substr(begin_offset, end_offset + 1 - begin_offset).str() + : ""; + + call_sites.push_back(CallSite{ + decl->getNameAsString(), + {mapping.to_uri(file_path), begin}, + func_content, + }); + } + std::string signature; + { + // begin loc + auto begin_loc = any_decl->getBeginLoc(); + auto [begin_fid, begin_offset] = mgr.getDecomposedLoc(begin_loc); + // body start or end loc + // auto body_start_loc = any_decl->getBody()->getBeginLoc(); + auto body = any_decl->getBody(); + auto end_loc = body ? body->getBeginLoc() : any_decl->getEndLoc(); + auto [end_fid, end_offset] = mgr.getDecomposedLoc(end_loc); + + if(begin_fid == end_fid && begin_offset <= end_offset) { + auto content = mgr.getBufferOrNone(begin_fid)->getBuffer(); + signature = + content.substr(begin_offset, end_offset + (body ? 0 : 1) - begin_offset) + .str(); + } + } + std::string func_content; + { + auto source_range = any_decl->getSourceRange(); + auto [begin_fid, begin_offset] = mgr.getDecomposedLoc(source_range.getBegin()); + auto [end_fid, end_offset] = mgr.getDecomposedLoc(source_range.getEnd()); + func_content = content.substr(begin_offset, end_offset + 1 - begin_offset).str(); + } + + CallGraphResult result; + + result.signature = std::move(signature); + result.content = std::move(func_content); + result.locations = std::move(locations); + result.callSites = std::move(call_sites); + + spdlog::details::registry::instance().flush_all(); + + co_return json::serialize(result); + } + } + + co_return json::Value{}; +} + } // namespace clice diff --git a/src/clice.cc b/src/clice.cc index f94e1298e..ebb012664 100644 --- a/src/clice.cc +++ b/src/clice.cc @@ -1,3 +1,5 @@ +#include "Server/Implement.h" +#include "Server/Plugin.h" #include "Server/Server.h" #include "Server/Version.h" #include "Support/Format.h" @@ -73,6 +75,32 @@ cl::opt log_level{ cl::desc("The log level, default is info"), }; +cl::opt> plugin_paths{ + "plugin-path", + cl::cat(category), + cl::value_desc("string"), + cl::init(std::vector{}), + cl::desc("The server plugins to load"), + cl::CommaSeparated, +}; + +/// Loads plugins intermediatly. +std::vector load_plugins(Server& instance) { + auto ref = ServerRef::Self{&instance}; + ServerPluginBuilder builder{ServerRef{&ref}}; + std::vector plugins; + for(auto& plugin_path: plugin_paths) { + auto plugin_instance = Plugin::load(plugin_path); + if(!plugin_instance) { + LOG_FATAL("Failed to load plugin {}: {}", plugin_path, plugin_instance.error()); + } + plugin_instance->register_server_callbacks(builder); + plugins.push_back(std::move(plugin_instance.value())); + } + // The llvm::sys::DynamicLibrary will be unloaded when the program exits. + return plugins; +} + } // namespace int main(int argc, const char** argv) { @@ -105,6 +133,10 @@ int main(int argc, const char** argv) { /// The global server instance. static Server instance; + + /// Loads plugins intermediatly before the server starts. + static auto plugins = load_plugins(instance); + auto loop = [&](json::Value value) -> async::Task<> { co_await instance.on_receive(value); };