diff --git a/.claude-plugin/.mcp.json b/.claude-plugin/.mcp.json index b1e81ed81..dd7d55d74 100644 --- a/.claude-plugin/.mcp.json +++ b/.claude-plugin/.mcp.json @@ -1,9 +1,5 @@ { "mempalace": { - "command": "python3", - "args": [ - "-m", - "mempalace.mcp_server" - ] + "command": "mempalace-mcp" } } diff --git a/.claude-plugin/hooks/mempal-precompact-hook.sh b/.claude-plugin/hooks/mempal-precompact-hook.sh index 0ac46ddc4..19bb6b0d4 100644 --- a/.claude-plugin/hooks/mempal-precompact-hook.sh +++ b/.claude-plugin/hooks/mempal-precompact-hook.sh @@ -1,5 +1,24 @@ #!/bin/bash # MemPalace PreCompact Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility -INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook precompact --harness claude-code +run_mempalace_hook() { + if command -v mempalace >/dev/null 2>&1; then + mempalace hook run "$@" + return $? + fi + + if command -v python3 >/dev/null 2>&1 && python3 -c "import mempalace" >/dev/null 2>&1; then + python3 -m mempalace hook run "$@" + return $? + fi + + if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then + python -m mempalace hook run "$@" + return $? + fi + + echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2 + return 1 +} + +run_mempalace_hook --hook precompact --harness claude-code diff --git a/.claude-plugin/hooks/mempal-stop-hook.sh b/.claude-plugin/hooks/mempal-stop-hook.sh index cba328496..5c860b47c 100644 --- a/.claude-plugin/hooks/mempal-stop-hook.sh +++ b/.claude-plugin/hooks/mempal-stop-hook.sh @@ -1,5 +1,24 @@ #!/bin/bash # MemPalace Stop Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility -INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook stop --harness claude-code +run_mempalace_hook() { + if command -v mempalace >/dev/null 2>&1; then + mempalace hook run "$@" + return $? + fi + + if command -v python3 >/dev/null 2>&1 && python3 -c "import mempalace" >/dev/null 2>&1; then + python3 -m mempalace hook run "$@" + return $? + fi + + if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then + python -m mempalace hook run "$@" + return $? + fi + + echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2 + return 1 +} + +run_mempalace_hook --hook stop --harness claude-code diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index be85be35c..aa15e5b26 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "mempalace", "source": "./.claude-plugin", "description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.", - "version": "3.3.2", + "version": "3.3.3", "author": { "name": "milla-jovovich" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a1a337721..a1b69a61d 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.3.2", + "version": "3.3.3", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" @@ -9,11 +9,7 @@ "commands": [], "mcpServers": { "mempalace": { - "command": "python3", - "args": [ - "-m", - "mempalace.mcp_server" - ] + "command": "mempalace-mcp" } }, "keywords": [ diff --git a/.codex-plugin/hooks/mempal-hook.sh b/.codex-plugin/hooks/mempal-hook.sh index 1cc005014..6d1113a41 100644 --- a/.codex-plugin/hooks/mempal-hook.sh +++ b/.codex-plugin/hooks/mempal-hook.sh @@ -3,7 +3,7 @@ set -euo pipefail HOOK_NAME="${1:?Usage: mempal-hook.sh }" INPUT_FILE=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; } cat > "$INPUT_FILE" -cat "$INPUT_FILE" | python3 -m mempalace hook run --hook "$HOOK_NAME" --harness codex +cat "$INPUT_FILE" | mempalace hook run --hook "$HOOK_NAME" --harness codex EXIT_CODE=$? rm -f "$INPUT_FILE" 2>/dev/null exit $EXIT_CODE diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 41d55fb21..16b66bb3a 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.3.2", + "version": "3.3.3", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" @@ -21,11 +21,7 @@ "hooks": "./hooks.json", "mcpServers": { "mempalace": { - "command": "python3", - "args": [ - "-m", - "mempalace.mcp_server" - ] + "command": "mempalace-mcp" } }, "interface": { diff --git a/CHANGELOG.md b/CHANGELOG.md index 483b8aa4d..2051ab360 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), --- +## [3.3.3] — 2026-04-23 + +### Bug Fixes + +- **Install regression** — `mempalace-mcp` console script is now declared in `pyproject.toml` alongside `.claude-plugin/plugin.json`'s reference to it. In v3.3.2 the two drifted apart (plugin.json shipped the new `"command": "mempalace-mcp"` form before the matching entry point landed), so every fresh `pip install mempalace==3.3.2` produced a Claude Code plugin config pointing at a binary that wasn't installed. (#1093, #340) +- Restore silent-save visibility after the Claude Code 2.1.114 client regression — production transcript saves were failing silently until this PR. (#1021) +- Paginate `status`-path metadata fetches so large palaces don't trip SQLite variable limits. (#851) +- Resolve the Claude plugin hook runner across platform / plugin-dir variations; previously broke on Windows and some macOS layouts. (#942) +- Real `python3` resolution for `.sh` hooks with a `MEMPAL_PYTHON` override path. (#833) +- Add optional `wing` parameter to `tool_diary_write` / `tool_diary_read` and derive per-project wing from the Claude Code transcript path when writing from the stop hook — diary entries from different projects no longer collapse into a shared default wing. (#659) +- Treat empty string as "no filter" in `mempalace_search` `wing`/`room`; LLM agents that default to filling every optional parameter with `""` no longer get bounced with `must be a non-empty string`. (#1097, #1084) +- Broaden `_wing_from_transcript_path` to handle Claude Code project folders without a `-Projects-` segment (e.g. `~/dev//`, `~/code/`). The project name is now derived from the final dash-separated token of the encoded folder, so Linux users with code outside `~/Projects/` get per-project diary scoping instead of falling through to `wing_sessions`. (#1145, follow-up to #659) +- `mempalace_diary_read(wing="")` now returns diary entries from every wing this agent has written to, matching the #1097 "empty-string as no filter" pattern. Previously defaulted to `wing_`, siloing entries that hooks wrote to project-derived wings. (#1145) +- `mempalace mine` now skips the generated `entities.json` file so its contents aren't re-ingested as project content. (#1175) + +### Improvements + +- **Deterministic hook saves.** Save hook now uses a silent Python API path, so successive hook invocations produce reproducible results and zero data loss on the hot path. (#673) +- **Graph cache with write-invalidation** inside `build_graph()` — warm-path calls no longer rebuild the palace-graph per request. (#661) +- **`mempalace init` entity detection overhaul.** Canonical project names now come from package manifests (`package.json`, `pyproject.toml`, `Cargo.toml`, `go.mod`) and real people come from git commit authors, rather than being inferred from prose. Includes union-find dedup across name/email aliases, bot filtering that keeps `@users.noreply.github.com` humans, and automatic "mine" flagging by contribution share. (#1148) +- **Regex detector accuracy.** CamelCase extraction so `MemPalace`, `ChromaDB`, `OpenAI` aren't fragmented; tighter versioned/hyphenated pattern kills `context-manager` / `multi-word` false positives; dialogue `^NAME:\s` requires ≥2 hits so `Created: ` metadata stops classifying field names as people; expanded stopwords for common English participles and descriptors; high-pronoun signal classifies as person rather than dumping to uncertain. (#1148) +- **Init → miner wire-up.** Confirmed entities merge into `~/.mempalace/known_entities.json` on init, which the miner reads to tag drawer metadata for entity-filtered search. Previously init's output was not consumed by the miner; the per-project `entities.json` is kept as an audit trail. (#1157) +- **Case-insensitive project dedup** across manifest, git, and convo sources so casing variants of the same project name collapse into one review entry. (#1175) + +### Added + +- i18n: Belarusian translation. (#1051) +- i18n: entity detection for German, Spanish, and French locales. (#1001) +- i18n: Traditional + Simplified Chinese entity detection. (#945) +- **`mempalace init --llm`**: optional LLM-assisted entity classification. Defaults to local Ollama (zero-API); also supports any OpenAI-compatible endpoint (LM Studio, llama.cpp server, vLLM, OpenRouter, etc.) and the Anthropic Messages API. Runs interactively with a progress indicator; Ctrl-C cancels cleanly and returns partial results. Useful for prose-heavy folders where the regex detector struggles (diaries, transcripts, research notes). Opt-in only — default init path remains zero-API. (#1150) +- **Claude Code conversation scanner.** `~/.claude/projects//` directories now contribute project entities using each session's authoritative `cwd` metadata, avoiding slug-decoding ambiguity. (#1150) + +### Known — deferred to v3.3.4 + +- HNSW parallel-insert SIGSEGV when `hnsw:num_threads` is unset on collection creation (#974) — fix in-flight as #976, awaiting rebase against develop. + +--- + ## [3.3.2] — 2026-04-19 ### Bug Fixes diff --git a/README.md b/README.md index 97cb4b006..acbeb143f 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). MIT — see [LICENSE](LICENSE). -[version-shield]: https://img.shields.io/badge/version-3.3.2-4dc9f6?style=flat-square&labelColor=0a0e14 +[version-shield]: https://img.shields.io/badge/version-3.3.3-4dc9f6?style=flat-square&labelColor=0a0e14 [release-link]: https://github.com/MemPalace/mempalace/releases [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8 [python-link]: https://www.python.org/ diff --git a/examples/HOOKS_TUTORIAL.md b/examples/HOOKS_TUTORIAL.md index 1b09467fd..3a34b81c2 100644 --- a/examples/HOOKS_TUTORIAL.md +++ b/examples/HOOKS_TUTORIAL.md @@ -25,4 +25,27 @@ Add this to your configuration file to enable automatic background saving: } ] } -} \ No newline at end of file +} +``` + +### 3. What changed (v3.1.0+) + +Both hooks now have **two-layer capture**: + +1. **Auto-mine**: Before blocking the AI, the hook runs the normalizer on the JSONL transcript and upserts chunks directly into the palace. This captures raw tool output (Bash results, search findings, build errors) that the AI would otherwise summarize away. + +2. **Updated reason messages**: The block reason now explicitly tells the AI to save tool output verbatim — not just topics and decisions. + +### 4. Backfill past conversations (one-time) + +The hooks capture conversations going forward, but you probably have months of past sessions. Run this once to mine them all: + +```bash +mempalace mine ~/.claude/projects/ --mode convos +``` + +### 5. Configuration + +- **`SAVE_INTERVAL=15`** — How many human messages between saves +- **`MEMPALACE_PYTHON`** — Python interpreter with mempalace + chromadb. Auto-detects: env var → repo venv → system python3 +- **`MEMPAL_DIR`** — Optional directory for auto-ingest via `mempalace mine` \ No newline at end of file diff --git a/examples/mcp_setup.md b/examples/mcp_setup.md index 9bc26dd61..2b7e1c3bf 100644 --- a/examples/mcp_setup.md +++ b/examples/mcp_setup.md @@ -5,13 +5,13 @@ Run the MCP server: ```bash -python -m mempalace.mcp_server +mempalace-mcp ``` Or add it to Claude Code: ```bash -claude mcp add mempalace -- python -m mempalace.mcp_server +claude mcp add mempalace -- mempalace-mcp ``` ## Available Tools diff --git a/hooks/README.md b/hooks/README.md index 977b109ed..7794527dd 100644 --- a/hooks/README.md +++ b/hooks/README.md @@ -6,10 +6,10 @@ These hook scripts make MemPalace save automatically. No manual "save" commands | Hook | When It Fires | What Happens | |------|--------------|-------------| -| **Save Hook** | Every 15 human messages | Blocks the AI, tells it to save key topics/decisions/quotes to the palace | -| **PreCompact Hook** | Right before context compaction | Emergency save — forces the AI to save EVERYTHING before losing context | +| **Save Hook** | Every 15 human messages | Auto-mines transcript (tool output included), then blocks the AI to save topics/decisions/quotes | +| **PreCompact Hook** | Right before context compaction | Auto-mines transcript, then emergency save — forces the AI to save EVERYTHING before losing context | -The AI does the actual filing — it knows the conversation context, so it classifies memories into the right wings/halls/closets. The hooks just tell it WHEN to save. +**Two-layer capture:** Hooks auto-mine the JSONL transcript directly into the palace (capturing raw tool output — Bash results, search findings, build errors). They also block the AI with a reason message telling it to save verbatim tool output and key context. Belt and suspenders — tool output gets stored even if the AI summarizes instead of quoting. ## Install — Claude Code @@ -68,6 +68,7 @@ Edit `mempal_save_hook.sh` to change: - **`SAVE_INTERVAL=15`** — How many human messages between saves. Lower = more frequent saves, higher = less interruption. - **`STATE_DIR`** — Where hook state is stored (defaults to `~/.mempalace/hook_state/`) - **`MEMPAL_DIR`** — Optional. Set to a conversations directory to auto-run `mempalace mine ` on each save trigger. Leave blank (default) to let the AI handle saving via the block reason message. +- **`MEMPALACE_PYTHON`** — Optional env var. Python interpreter with mempalace + chromadb installed. Auto-detects: `MEMPALACE_PYTHON` env var → repo `venv/bin/python3` → system `python3`. Set this if your venv is in a non-standard location. ### mempalace CLI @@ -91,15 +92,19 @@ User sends message → AI responds → Claude Code fires Stop hook ↓ ┌─── < 15 since last save ──→ echo "{}" (let AI stop) │ - └─── ≥ 15 since last save ──→ {"decision": "block", "reason": "save..."} - ↓ - AI saves to palace - ↓ - AI tries to stop again - ↓ - stop_hook_active = true - ↓ - Hook sees flag → echo "{}" (let it through) + └─── ≥ 15 since last save + ↓ + Auto-mine transcript → palace (tool output captured) + ↓ + {"decision": "block", "reason": "save tool output verbatim..."} + ↓ + AI saves to palace (topics, decisions, quotes) + ↓ + AI tries to stop again + ↓ + stop_hook_active = true + ↓ + Hook sees flag → echo "{}" (let it through) ``` The `stop_hook_active` flag prevents infinite loops: block once → AI saves → tries to stop → flag is true → we let it through. @@ -109,14 +114,18 @@ The `stop_hook_active` flag prevents infinite loops: block once → AI saves → ``` Context window getting full → Claude Code fires PreCompact ↓ - Hook ALWAYS blocks + Find transcript (from input or session_id lookup) + ↓ + Auto-mine transcript → palace (tool output captured) + ↓ + {"decision": "block", "reason": "save tool output verbatim..."} ↓ AI saves everything ↓ Compaction proceeds ``` -No counting needed — compaction always warrants a save. +No counting needed — compaction always warrants a save. The auto-mine captures raw tool output before the AI gets a chance to summarize it away. ## Debugging @@ -137,6 +146,36 @@ Example output: **Hooks require session restart after install.** Claude Code loads hooks from `settings.json` at session start only. If you run `mempalace init` or manually edit hook config mid-session, the hooks won't fire until you restart Claude Code. This is a Claude Code limitation. +**`MEMPAL_PYTHON` override for the hook's internal Python calls.** The save hook parses its JSON input and counts transcript messages with `python3`. When the harness is launched from a GUI on macOS — `open -a`, Spotlight, the dock — its `PATH` is the minimal `/usr/bin:/bin:/usr/sbin:/sbin` inherited from `launchd`, not your shell PATH. If `python3` isn't on that PATH, those internal calls fail and the hook can't count exchanges. + +Point the hook at any Python 3 interpreter to fix it: + +```bash +export MEMPAL_PYTHON="/usr/bin/python3" # system Python is fine +export MEMPAL_PYTHON="$HOME/.venvs/mempalace/bin/python" # or your venv +``` + +Resolution priority: `$MEMPAL_PYTHON` (if set and executable) → `$(command -v python3)` → bare `python3`. The interpreter only needs `json` and `sys` from the standard library — `mempalace` itself does not need to be installed in it. + +Note: the `mempalace mine` auto-ingest runs via the `mempalace` CLI, so that command also needs to be on the hook's `PATH`. Installing with `pipx install mempalace` or `uv tool install mempalace` puts it on a stable global location; otherwise extend the hook environment's `PATH` to include your venv's `bin/`. + +## Backfill Past Conversations + +The hooks only capture conversations going forward. To mine **past** Claude Code sessions into your palace, run a one-time backfill: + +```bash +mempalace mine ~/.claude/projects/ --mode convos +``` + +This scans all JSONL transcripts from previous sessions and files them into the `conversations` wing. On a typical developer machine with months of history, this can yield 50K–200K drawers. + +For Codex CLI sessions: +```bash +mempalace mine ~/.codex/sessions/ --mode convos +``` + +This only needs to be done once — after that, the hooks auto-mine each session as you go. + ## Cost **Zero extra tokens.** The hooks notify the AI that saves happened in the background — the AI doesn't need to write anything in the chat. All filing is handled automatically. Previous versions asked the AI to write diary entries and drawer content in the chat window, which cost ~$1/session in retransmitted tokens. diff --git a/hooks/mempal_precompact_hook.sh b/hooks/mempal_precompact_hook.sh index 3a74ca54e..a14a0d0e9 100755 --- a/hooks/mempal_precompact_hook.sh +++ b/hooks/mempal_precompact_hook.sh @@ -54,10 +54,17 @@ mkdir -p "$STATE_DIR" # Leave empty to skip auto-ingest (AI handles saving via the block reason). MEMPAL_DIR="" +# Resolve the Python interpreter. Same contract as mempal_save_hook.sh: +# MEMPAL_PYTHON (explicit override) → $(command -v python3) → bare python3. +MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}" +if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then + MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)" +fi + # Read JSON input from stdin INPUT=$(cat) -SESSION_ID=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null) +SESSION_ID=$(echo "$INPUT" | "$MEMPAL_PYTHON_BIN" -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null) echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$STATE_DIR/hook.log" @@ -65,7 +72,7 @@ echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$ if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$SCRIPT_DIR")" - python3 -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 + mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 fi # Silent: return empty JSON to not block. "decision": "allow" is invalid — diff --git a/hooks/mempal_save_hook.sh b/hooks/mempal_save_hook.sh index d8e7cc248..228b41bac 100755 --- a/hooks/mempal_save_hook.sh +++ b/hooks/mempal_save_hook.sh @@ -61,13 +61,30 @@ mkdir -p "$STATE_DIR" # Leave empty to skip auto-ingest (AI handles saving via the block reason). MEMPAL_DIR="" +# Resolve the Python interpreter the hook should use. +# +# Why this is nontrivial: GUI-launched Claude Code on macOS (or any harness +# that doesn't inherit the user's shell PATH) may find a `python3` on PATH +# that lacks mempalace — e.g. /usr/bin/python3 while the user installed +# mempalace into a venv or pyenv. Users in that situation can point the +# hook at the right interpreter by exporting MEMPAL_PYTHON. +# +# Resolution order (first hit wins): +# 1. $MEMPAL_PYTHON — explicit user override (absolute path) +# 2. $(command -v python3) — first python3 on the hook's PATH +# 3. bare "python3" — last-resort fallback (hope the PATH has it) +MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}" +if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then + MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)" +fi + # Read JSON input from stdin INPUT=$(cat) # Parse all fields in a single Python call (3x faster than separate invocations) # SECURITY: All values are sanitized before being interpolated into shell assignments. # stop_hook_active is coerced to a strict True/False to prevent command injection via eval. -eval $(echo "$INPUT" | python3 -c " +eval $(echo "$INPUT" | "$MEMPAL_PYTHON_BIN" -c " import sys, json, re data = json.load(sys.stdin) sid = data.get('session_id', 'unknown') @@ -95,7 +112,7 @@ fi # Count human messages in the JSONL transcript # SECURITY: Pass transcript path as sys.argv to avoid shell injection via crafted paths if [ -f "$TRANSCRIPT_PATH" ]; then - EXCHANGE_COUNT=$(python3 - "$TRANSCRIPT_PATH" <<'PYEOF' + EXCHANGE_COUNT=$("$MEMPAL_PYTHON_BIN" - "$TRANSCRIPT_PATH" <<'PYEOF' import json, sys count = 0 with open(sys.argv[1]) as f: @@ -144,7 +161,6 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then # 1. TRANSCRIPT_PATH (from Claude Code) — mine the directory it lives in # 2. MEMPAL_DIR (user-configured) — mine that directory # At least one should work. If neither is set, nothing mines. - PYTHON="$(command -v python3)" MINE_DIR="" if [ -n "$TRANSCRIPT_PATH" ] && [ -f "$TRANSCRIPT_PATH" ]; then MINE_DIR="$(dirname "$TRANSCRIPT_PATH")" @@ -153,7 +169,7 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then MINE_DIR="$MEMPAL_DIR" fi if [ -n "$MINE_DIR" ]; then - "$PYTHON" -m mempalace mine "$MINE_DIR" >> "$STATE_DIR/hook.log" 2>&1 & + mempalace mine "$MINE_DIR" >> "$STATE_DIR/hook.log" 2>&1 & fi # MEMPAL_VERBOSE toggle: diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 1a171c167..3a0d2c3f9 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -120,8 +120,7 @@ def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 3600.0) -> li os.rename(seg_dir, target) moved.append(target) logger.warning( - "Quarantined stale HNSW segment %s " - "(sqlite %.0fs newer than HNSW); renamed to %s", + "Quarantined stale HNSW segment %s (sqlite %.0fs newer than HNSW); renamed to %s", seg_dir, sqlite_mtime - hnsw_mtime, target, diff --git a/mempalace/cli.py b/mempalace/cli.py index a4c8a886c..714c64c7b 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -71,7 +71,8 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool: def cmd_init(args): import json from pathlib import Path - from .entity_detector import scan_for_detection, detect_entities, confirm_entities + from .entity_detector import confirm_entities + from .project_scanner import discover_entities from .room_detector_local import detect_rooms_local cfg = MempalaceConfig() @@ -85,25 +86,55 @@ def cmd_init(args): languages = cfg.entity_languages languages_tuple = tuple(languages) - # Pass 1: auto-detect people and projects from file content + # Optional phase-2 LLM provider (opt-in via --llm). + llm_provider = None + if getattr(args, "llm", False): + from .llm_client import LLMError, get_provider + + try: + llm_provider = get_provider( + name=args.llm_provider, + model=args.llm_model, + endpoint=args.llm_endpoint, + api_key=args.llm_api_key, + ) + except LLMError as e: + print(f" ERROR: {e}", file=sys.stderr) + sys.exit(2) + ok, msg = llm_provider.check_available() + if not ok: + print( + f" ERROR: LLM provider '{args.llm_provider}' unavailable: {msg}", + file=sys.stderr, + ) + sys.exit(2) + print(f" LLM refinement enabled: {args.llm_provider}/{args.llm_model}") + + # Pass 1: discover entities — manifests + git authors first, prose detection + # as supplement for names mentioned only in docs/notes. Optional phase-2 + # LLM refinement runs inside discover_entities when llm_provider is given. print(f"\n Scanning for entities in: {args.dir}") if languages_tuple != ("en",): print(f" Languages: {', '.join(languages_tuple)}") - files = scan_for_detection(args.dir) - if files: - print(f" Reading {len(files)} files...") - detected = detect_entities(files, languages=languages_tuple) - total = len(detected["people"]) + len(detected["projects"]) + len(detected["uncertain"]) - if total > 0: - confirmed = confirm_entities(detected, yes=getattr(args, "yes", False)) - # Save confirmed entities to /entities.json for the miner - if confirmed["people"] or confirmed["projects"]: - entities_path = Path(args.dir).expanduser().resolve() / "entities.json" - with open(entities_path, "w") as f: - json.dump(confirmed, f, indent=2) - print(f" Entities saved: {entities_path}") - else: - print(" No entities detected — proceeding with directory-based rooms.") + detected = discover_entities(args.dir, languages=languages_tuple, llm_provider=llm_provider) + total = len(detected["people"]) + len(detected["projects"]) + len(detected["uncertain"]) + if total > 0: + confirmed = confirm_entities(detected, yes=getattr(args, "yes", False)) + # Save confirmed entities to /entities.json (per-project + # audit trail — user can inspect or hand-edit) AND merge into the + # global registry the miner reads at mine time. + if confirmed["people"] or confirmed["projects"]: + entities_path = Path(args.dir).expanduser().resolve() / "entities.json" + with open(entities_path, "w", encoding="utf-8") as f: + json.dump(confirmed, f, indent=2, ensure_ascii=False) + print(f" Entities saved: {entities_path}") + + from .miner import add_to_known_entities + + registry_path = add_to_known_entities(confirmed) + print(f" Registry updated: {registry_path}") + else: + print(" No entities detected — proceeding with directory-based rooms.") # Pass 2: detect rooms from folder structure detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False)) @@ -367,7 +398,7 @@ def cmd_instructions(args): def cmd_mcp(args): """Show how to wire MemPalace into MCP-capable hosts.""" - base_server_cmd = "python -m mempalace.mcp_server" + base_server_cmd = "mempalace-mcp" if args.palace: resolved_palace = str(Path(args.palace).expanduser()) @@ -551,6 +582,43 @@ def main(): "When given, the value is also persisted to config.json." ), ) + p_init.add_argument( + "--llm", + action="store_true", + help=( + "Enable LLM-assisted entity refinement (opt-in, local-first). " + "Runs after manifest/git/regex detection, asking the configured " + "provider to reclassify ambiguous candidates. " + "Ctrl-C during refinement returns partial results." + ), + ) + p_init.add_argument( + "--llm-provider", + default="ollama", + choices=["ollama", "openai-compat", "anthropic"], + help="LLM provider (default: ollama). Use --llm to enable.", + ) + p_init.add_argument( + "--llm-model", + default="gemma4:e4b", + help="Model name for the chosen provider (default: gemma4:e4b for Ollama).", + ) + p_init.add_argument( + "--llm-endpoint", + default=None, + help=( + "Provider endpoint URL. Default for Ollama: http://localhost:11434. " + "Required for openai-compat." + ), + ) + p_init.add_argument( + "--llm-api-key", + default=None, + help=( + "API key for the provider. For anthropic, defaults to $ANTHROPIC_API_KEY; " + "for openai-compat, defaults to $OPENAI_API_KEY." + ), + ) # mine p_mine = sub.add_parser("mine", help="Mine files into the palace") diff --git a/mempalace/config.py b/mempalace/config.py index a9bcc7ffb..616334e5c 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -168,7 +168,10 @@ def palace_path(self): """Path to the memory palace data directory.""" env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH") if env_val: - return env_val + # Normalize: expand ~ and collapse .. to match the CLI --palace + # code path (mcp_server.py:62) and prevent surprise redirection + # when the env var contains unresolved components. + return os.path.abspath(os.path.expanduser(env_val)) return self._file_config.get("palace_path", DEFAULT_PALACE_PATH) @property diff --git a/mempalace/convo_scanner.py b/mempalace/convo_scanner.py new file mode 100644 index 000000000..b592494d6 --- /dev/null +++ b/mempalace/convo_scanner.py @@ -0,0 +1,160 @@ +""" +convo_scanner.py — Parse Claude Code conversation directories into ProjectInfo. + +Claude Code stores sessions under ``~/.claude/projects//.jsonl``, +where the ```` is the original CWD with ``/`` replaced by ``-``. That +encoding is lossy: we can't tell whether ``foo-bar`` in a slug is the +literal project name ``foo-bar`` or two path segments ``foo/bar``. + +Fortunately, every message record in the JSONL carries a ``cwd`` field with +the true path. This scanner reads one record per session to recover the +accurate project name, falling back to slug-decoding only if the JSONL +is malformed or empty. + +Output is the same ``ProjectInfo`` shape used by ``project_scanner``, so the +``discover_entities`` orchestrator can mix-and-match sources. + +Public: + is_claude_projects_root(path) -> bool + scan_claude_projects(path) -> list[ProjectInfo] +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Optional + +from mempalace.project_scanner import ProjectInfo + + +MAX_HEADER_LINES = 20 # lines to read per session looking for `cwd` + + +def is_claude_projects_root(path: Path) -> bool: + """Return True if path looks like `.claude/projects/`. + + Heuristic: at least one child dir whose name starts with ``-`` and which + contains at least one ``.jsonl`` file. + """ + if not path.is_dir(): + return False + try: + children = list(path.iterdir()) + except OSError: + return False + for child in children: + if not (child.is_dir() and child.name.startswith("-")): + continue + try: + if any(p.suffix == ".jsonl" for p in child.iterdir() if p.is_file()): + return True + except OSError: + continue + return False + + +def _extract_cwd_from_session(session_file: Path) -> Optional[str]: + """Return the ``cwd`` from the first message record that carries one. + + Returns None if the file can't be read, has no JSON, or no record has cwd. + """ + try: + with open(session_file, encoding="utf-8", errors="replace") as f: + for i, line in enumerate(f): + if i >= MAX_HEADER_LINES: + break + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + cwd = obj.get("cwd") + if isinstance(cwd, str) and cwd: + return cwd + except OSError: + return None + return None + + +def _decode_slug_fallback(slug: str) -> str: + """Best-effort project name from slug when cwd is unavailable. + + The slug is lossy (`/` and `-` both become `-`). Last non-empty segment + is the closest guess at the project name, preserving kebab-case is + impossible without cwd. + """ + stripped = slug.lstrip("-") + parts = [p for p in stripped.split("-") if p] + return parts[-1] if parts else slug + + +def _safe_mtime(path: Path) -> float: + """Return file mtime, defaulting old on permission or filesystem errors.""" + try: + return path.stat().st_mtime + except OSError: + return 0.0 + + +def _resolve_project_name(project_dir: Path) -> str: + """Read one session's cwd to recover the original project name. + + Falls back to slug-decoding if no session has a readable cwd. + """ + sessions = sorted( + (p for p in project_dir.iterdir() if p.is_file() and p.suffix == ".jsonl"), + key=_safe_mtime, + reverse=True, # newest first — most likely to be well-formed + ) + for session in sessions: + cwd = _extract_cwd_from_session(session) + if cwd: + return Path(cwd).name or cwd + return _decode_slug_fallback(project_dir.name) + + +def scan_claude_projects(path: str | Path) -> list[ProjectInfo]: + """Scan a ``.claude/projects/`` directory for Claude Code conversations. + + One ProjectInfo per subdir. ``has_git`` is False (the directory isn't a + repo itself) but ``total_commits`` is repurposed here as session count so + the UX surfaces a density signal for ranking. + """ + root = Path(path).expanduser().resolve() + if not is_claude_projects_root(root): + return [] + + projects: dict[str, ProjectInfo] = {} + for sub in sorted(root.iterdir()): + if not (sub.is_dir() and sub.name.startswith("-")): + continue + try: + sessions = [p for p in sub.iterdir() if p.is_file() and p.suffix == ".jsonl"] + except OSError: + continue + if not sessions: + continue + + name = _resolve_project_name(sub) + session_count = len(sessions) + + proj = ProjectInfo( + name=name, + repo_root=sub, + manifest=None, + has_git=False, + total_commits=session_count, + user_commits=session_count, + is_mine=True, # Claude Code sessions are authored by the user + ) + existing = projects.get(name) + if existing is None or session_count > existing.user_commits: + projects[name] = proj + + return sorted( + projects.values(), + key=lambda p: (-p.user_commits, p.name), + ) diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index 754c65dce..2f2aae481 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -113,6 +113,23 @@ def _get_stopwords(languages: tuple) -> frozenset: ".next", "coverage", ".mempalace", + ".terraform", + "vendor", + "target", +} + +# Files whose content is boilerplate prose — poisons entity detection. +# Matched by stem (case-insensitive), with or without an extension. +SKIP_FILENAMES = { + "license", + "licence", + "copying", + "copyright", + "notice", + "authors", + "patents", + "third_party_notices", + "third-party-notices", } @@ -193,7 +210,7 @@ def _compile_each(raw_patterns, flags=re.IGNORECASE): "person_verbs": _compile_each(sources["person_verb_patterns"]), "project_verbs": _compile_each(sources["project_verb_patterns"]), "direct": direct_compiled, - "versioned": re.compile(rf"\b{n}[-v]\w+", re.IGNORECASE), + "versioned": re.compile(rf"\b{n}[-_]v?\d+(?:\.\d+)*\b", re.IGNORECASE), "code_ref": re.compile(rf"\b{n}\.(py|js|ts|yaml|yml|json|sh)\b", re.IGNORECASE), } @@ -227,12 +244,19 @@ def score_entity(name: str, text: str, lines: list, languages=("en",)) -> dict: # --- Person signals --- - # Dialogue markers (strong signal) + # Dialogue markers (strong signal). + # The bare `^NAME:\s` colon-prefix pattern matches metadata lines like + # `Created: 2026-04-21`, so we require >= 2 hits for it to count as dialogue + # (real speaker markers repeat; single-line metadata doesn't). for rx in patterns["dialogue"]: matches = len(rx.findall(text)) - if matches > 0: - person_score += matches * 3 - person_signals.append(f"dialogue marker ({matches}x)") + if matches == 0: + continue + is_bare_colon = rx.pattern.endswith(r":\s") and not rx.pattern.endswith(r"[:\s]") + if is_bare_colon and matches < 2: + continue + person_score += matches * 3 + person_signals.append(f"dialogue marker ({matches}x)") # Person verbs for rx in patterns["person_verbs"]: @@ -328,17 +352,28 @@ def classify_entity(name: str, frequency: int, scores: dict) -> dict: signal_categories.add("addressed") has_two_signal_types = len(signal_categories) >= 2 - _ = signal_categories - {"pronoun"} # reserved for future thresholds + # Single-category pronoun signal still classifies as person when the + # evidence is overwhelming — a diary's main character is referenced + # with pronouns, not dialogue markers. Require both: many pronoun hits + # AND a high pronoun-to-frequency ratio so common sentence-start words + # (Never, Before, etc.) with incidental pronoun proximity don't qualify. + pronoun_hits = 0 + for s in scores["person_signals"]: + m = re.search(r"pronoun nearby \((\d+)x\)", s) + if m: + pronoun_hits = int(m.group(1)) + break + strong_pronoun_signal = pronoun_hits >= 5 and frequency > 0 and pronoun_hits / frequency >= 0.2 - if person_ratio >= 0.7 and has_two_signal_types and ps >= 5: + if person_ratio >= 0.7 and (has_two_signal_types and ps >= 5 or strong_pronoun_signal): entity_type = "person" confidence = min(0.99, 0.5 + person_ratio * 0.5) signals = scores["person_signals"] or [f"appears {frequency}x"] - elif person_ratio >= 0.7 and (not has_two_signal_types or ps < 5): - # Pronoun-only match — downgrade to uncertain + elif person_ratio >= 0.7: + # Weak single-category person signal — downgrade to uncertain entity_type = "uncertain" confidence = 0.4 - signals = scores["person_signals"] + [f"appears {frequency}x — pronoun-only match"] + signals = scores["person_signals"] + [f"appears {frequency}x — weak person signal"] elif person_ratio <= 0.3: entity_type = "project" confidence = min(0.99, 0.5 + (1 - person_ratio) * 0.5) @@ -560,6 +595,8 @@ def scan_for_detection(project_dir: str, max_files: int = 10) -> list: dirs[:] = [d for d in dirs if d not in SKIP_DIRS] for filename in filenames: filepath = Path(root) / filename + if filepath.stem.lower() in SKIP_FILENAMES: + continue ext = filepath.suffix.lower() if ext in PROSE_EXTENSIONS: prose_files.append(filepath) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 92184f552..01eca3fb4 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -17,22 +17,54 @@ SAVE_INTERVAL = 15 STATE_DIR = Path.home() / ".mempalace" / "hook_state" + +def _mempalace_python() -> str: + """Return the python interpreter that has mempalace installed. + + When hooks are invoked by Claude Code, sys.executable may be the system + python which lacks chromadb and other deps. Resolution order: + 1. MEMPALACE_PYTHON env var (explicit override) + 2. Venv python from package install path + 3. Editable install: venv/ sibling to mempalace/ + 4. sys.executable fallback + """ + # Honor explicit override (used by shell hook wrappers) + env_python = os.environ.get("MEMPALACE_PYTHON", "") + if env_python and os.path.isfile(env_python) and os.access(env_python, os.X_OK): + return env_python + # This file lives at /lib/pythonX.Y/site-packages/mempalace/hooks_cli.py + # or /mempalace/hooks_cli.py (editable install). + venv_bin = Path(__file__).resolve().parents[3] / "bin" / "python" + if venv_bin.is_file(): + return str(venv_bin) + # Editable install: assumes project root has a venv/ sibling to mempalace/ + project_venv = Path(__file__).resolve().parents[1] / "venv" / "bin" / "python" + if project_venv.is_file(): + return str(project_venv) + return sys.executable + + +_RECENT_MSG_COUNT = 30 # how many recent user messages to summarize + STOP_BLOCK_REASON = ( "AUTO-SAVE checkpoint (MemPalace). Save this session's key content:\n" - "1. mempalace_diary_write — AAAK-compressed session summary\n" - "2. mempalace_add_drawer — verbatim quotes, decisions, code snippets\n" + "1. mempalace_diary_write — session summary (what was discussed, " + "key decisions, current state of work)\n" + "2. mempalace_add_drawer — verbatim quotes, decisions, code snippets " + "(place in appropriate wing and room)\n" "3. mempalace_kg_add — entity relationships (optional)\n" - "Do NOT write to Claude Code's native auto-memory (.md files). " - "Continue conversation after saving." + "For THIS save, use MemPalace MCP tools only (not auto-memory .md files). " + "Use verbatim quotes where possible. Continue conversation after saving." ) PRECOMPACT_BLOCK_REASON = ( "COMPACTION IMMINENT (MemPalace). Save ALL session content before context is lost:\n" - "1. mempalace_diary_write — thorough AAAK-compressed session summary\n" - "2. mempalace_add_drawer — ALL verbatim quotes, decisions, code, context\n" + "1. mempalace_diary_write — thorough session summary\n" + "2. mempalace_add_drawer — ALL verbatim quotes, decisions, code, context " + "(place each in appropriate wing and room)\n" "3. mempalace_kg_add — entity relationships (optional)\n" - "Be thorough \u2014 after compaction, detailed context will be lost. " - "Do NOT write to Claude Code's native auto-memory (.md files). " + "For THIS save, use MemPalace MCP tools only (not auto-memory .md files). " + "Be thorough — after compaction this is all that survives. " "Save everything to MemPalace, then allow compaction to proceed." ) @@ -134,8 +166,35 @@ def _log(message: str): def _output(data: dict): - """Print JSON to stdout with consistent formatting (pretty-printed).""" - print(json.dumps(data, indent=2, ensure_ascii=False)) + """Print JSON to stdout without importing modules that may redirect streams. + + If mempalace.mcp_server is already loaded, reuse its saved real stdout fd. + Otherwise, write directly to fd 1 so hook responses still go to stdout even + if sys.stdout has been redirected elsewhere. + """ + payload = (json.dumps(data, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + + real_stdout_fd: int | None = None + mcp_mod = sys.modules.get("mempalace.mcp_server") or sys.modules.get( + f"{__package__}.mcp_server" if __package__ else "mcp_server" + ) + if mcp_mod is not None: + real_stdout_fd = getattr(mcp_mod, "_REAL_STDOUT_FD", None) + + fd = real_stdout_fd if real_stdout_fd is not None else 1 + offset = 0 + try: + while offset < len(payload): + try: + offset += os.write(fd, payload[offset:]) + except InterruptedError: + continue + return + except OSError: + pass + + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() def _get_mine_dir(transcript_path: str = "") -> str: @@ -237,6 +296,182 @@ def _mine_sync(transcript_path: str = ""): pass +def _desktop_toast(body: str, title: str = "MemPalace"): + """Send a desktop notification via notify-send. Fails silently.""" + try: + subprocess.Popen( + ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError: + pass + + +def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUNT) -> list[str]: + """Extract the last N user messages from a JSONL transcript.""" + path = Path(transcript_path).expanduser() + if not path.is_file(): + return [] + messages = [] + try: + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + try: + entry = json.loads(line) + # Claude Code format + msg = entry.get("message") or entry.get("event_message") or {} + if isinstance(msg, dict) and msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, list): + content = " ".join( + b.get("text", "") for b in content if isinstance(b, dict) + ) + if not isinstance(content, str) or not content.strip(): + continue + if "" in content or "" in content: + continue + messages.append(content.strip()[:200]) + # Codex CLI format + elif entry.get("type") == "event_msg": + payload = entry.get("payload", {}) + if isinstance(payload, dict) and payload.get("type") == "user_message": + text = payload.get("message", "") + if isinstance(text, str) and text.strip(): + if "" not in text: + messages.append(text.strip()[:200]) + except (json.JSONDecodeError, AttributeError): + pass + except OSError: + return [] + return messages[-count:] + + +_THEME_STOPWORDS = frozenset( + "the a an and or but in on at to for of is it i me my you your we our " + "this that with from by was were be been are not no yes can do did dont " + "will would should could have has had lets let just also like so if then " + "ok okay sure yeah hey hi here there what when where how why which some " + "all any each every about into out up down over after before between " + "get got make made need want use used using check look see run try " + "know think right now still already really very much more most too " + "file files code one two new first last next thing things way well".split() +) + + +def _extract_themes(messages: list[str], max_themes: int = 3) -> list[str]: + """Pull 2-3 distinctive topic words from recent messages. + + Note: stopword list is English-only; non-English corpora will produce noisy themes. + """ + from collections import Counter + + words: Counter[str] = Counter() + for msg in messages: + for word in msg.lower().split(): + # Strip punctuation, keep words 4+ chars + clean = word.strip(".,;:!?\"'`()[]{}#<>/\\-_=+@$%^&*~") + if len(clean) >= 4 and clean not in _THEME_STOPWORDS and clean.isalpha(): + words[clean] += 1 + return [w for w, _ in words.most_common(max_themes)] + + +def _save_diary_direct( + transcript_path: str, + session_id: str, + wing: str = "", + toast: bool = False, +) -> dict: + """Write a diary checkpoint by calling the tool function directly (no MCP roundtrip). + + If `wing` is set, the entry lands in that wing (typically the project wing + derived from the transcript path). Otherwise falls back to `tool_diary_write`'s + default of `wing_session-hook`. + + Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure. + """ + messages = _extract_recent_messages(transcript_path) + if not messages: + _log("No recent messages to save") + return {"count": 0} + + themes = _extract_themes(messages) + + # Build a compressed diary entry from recent conversation + now = datetime.now() + topics = "|".join(m[:80] for m in messages[-10:]) + entry = ( + f"CHECKPOINT:{now.strftime('%Y-%m-%d')}|session:{session_id}" + f"|msgs:{len(messages)}|recent:{topics}" + ) + + try: + from .mcp_server import tool_diary_write + + result = tool_diary_write( + agent_name="session-hook", + entry=entry, + topic="checkpoint", + wing=wing, + ) + if result.get("success"): + _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") + # Write state for ack tool to read + try: + ack_file = STATE_DIR / "last_checkpoint" + ack_file.write_text( + json.dumps({"msgs": len(messages), "ts": now.isoformat()}), + encoding="utf-8", + ) + except OSError: + pass + if toast: + _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived") + return {"count": len(messages), "themes": themes} + else: + _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") + except Exception as e: + _log(f"Diary checkpoint error: {e}") + return {"count": 0} + + +def _ingest_transcript(transcript_path: str): + """Mine a Claude Code session transcript into the palace as a conversation.""" + path = Path(transcript_path).expanduser() + if not path.is_file() or path.stat().st_size < 100: + return + + from .config import MempalaceConfig + + try: + MempalaceConfig() # validate config loads + except Exception: + return + + try: + log_path = STATE_DIR / "hook.log" + STATE_DIR.mkdir(parents=True, exist_ok=True) + with open(log_path, "a") as log_f: + subprocess.Popen( + [ + _mempalace_python(), + "-m", + "mempalace", + "mine", + str(path.parent), + "--mode", + "convos", + "--wing", + "sessions", + ], + stdout=log_f, + stderr=log_f, + ) + _log(f"Transcript ingest started: {path.name}") + except OSError: + pass + + SUPPORTED_HARNESSES = {"claude-code", "codex"} @@ -252,6 +487,39 @@ def _parse_harness_input(data: dict, harness: str) -> dict: } +def _wing_from_transcript_path(transcript_path: str) -> str: + """Derive a project wing name from a Claude Code transcript path. + + Claude Code encodes the project's source directory by replacing path + separators with dashes, producing folders like: + ~/.claude/projects/-home--Projects-/session.jsonl + ~/.claude/projects/-home--dev--/session.jsonl + ~/.claude/projects/-Users---/session.jsonl + + The project directory name is the final dash-separated token of the + encoded folder. Returns ``wing_`` (lowercased, spaces → ``_``). + Falls back to ``wing_sessions`` if the path does not match a Claude Code + project-folder layout. + """ + # Normalize path separators for cross-platform (Windows backslashes) + normalized = transcript_path.replace("\\", "/") + # Primary: pull the encoded project folder out of ``.claude/projects/`` + # and take its last dash-separated token. + match = re.search(r"/\.claude/projects/-([^/]+)", normalized) + if match: + encoded = match.group(1) + project = encoded.rsplit("-", 1)[-1] + if project: + return f"wing_{project.lower().replace(' ', '_')}" + # Legacy fallback: explicit ``-Projects-`` segment, useful for + # transcripts not under the standard Claude Code projects dir. + match = re.search(r"-Projects-([^/]+?)(?:/|$)", normalized) + if match: + project = match.group(1).lower().replace(" ", "_") + return f"wing_{project}" + return "wing_sessions" + + def hook_stop(data: dict, harness: str): """Stop hook: block every N messages for auto-save.""" parsed = _parse_harness_input(data, harness) @@ -259,10 +527,29 @@ def hook_stop(data: dict, harness: str): stop_hook_active = parsed["stop_hook_active"] transcript_path = parsed["transcript_path"] - # If already in a save cycle, let through (infinite-loop prevention) + # If already in a block-mode save cycle, let through (infinite-loop prevention). + # Silent mode saves directly without returning {"decision":"block"}, so there's + # no loop to prevent — and Claude Code's plugin dispatch sets this flag on every + # fire after the first, which would otherwise suppress all subsequent auto-saves. if str(stop_hook_active).lower() in ("true", "1", "yes"): - _output({}) - return + # Safe default: assume silent mode on any config-read failure so saves + # proceed rather than being silently dropped. Silent mode is the default + # (v3.3.0+), so if we can't read config, behave as if it's still on. + silent_guard = True + try: + from .config import MempalaceConfig + except ImportError as exc: + _log( + f"WARNING: could not import MempalaceConfig for stop guard: {exc}; defaulting to silent mode" + ) + else: + try: + silent_guard = MempalaceConfig().hook_silent_save + except AttributeError as exc: + _log(f"WARNING: could not read hook_silent_save: {exc}; defaulting to silent mode") + if not silent_guard: + _output({}) + return # Count human messages exchange_count = _count_human_messages(transcript_path) @@ -282,18 +569,62 @@ def hook_stop(data: dict, harness: str): _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save") if since_last >= SAVE_INTERVAL and exchange_count > 0: - # Update last save point - try: - last_save_file.write_text(str(exchange_count), encoding="utf-8") - except OSError: - pass - _log(f"TRIGGERING SAVE at exchange {exchange_count}") - # Optional: auto-ingest if MEMPAL_DIR is set - _maybe_auto_ingest(transcript_path) + # Read hook settings from config + from .config import MempalaceConfig - _output({"decision": "block", "reason": STOP_BLOCK_REASON}) + try: + config = MempalaceConfig() + silent = config.hook_silent_save + toast = config.hook_desktop_toast + except Exception: + silent = True + toast = False + + project_wing = _wing_from_transcript_path(transcript_path) + + if silent: + # Save directly via Python API — systemMessage renders in terminal + result = {"count": 0} + if transcript_path: + result = _save_diary_direct( + transcript_path, session_id, wing=project_wing, toast=toast + ) + _ingest_transcript(transcript_path) + _maybe_auto_ingest(transcript_path) + # Only advance save marker after successful save + count = result.get("count", 0) + if count > 0: + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass + themes = result.get("themes", []) + if themes: + tag = " \u2014 " + ", ".join(themes) + else: + tag = "" + _output( + { + "systemMessage": f"\u2726 {count} memories woven into the palace{tag}", + } + ) + else: + _output({}) + else: + # Legacy: block and ask Claude to save via MCP tools. + # Marker advances before confirmed save — best-effort; if Claude + # fails to save, the checkpoint is lost but won't retry endlessly. + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass + if transcript_path: + _ingest_transcript(transcript_path) + _maybe_auto_ingest(transcript_path) + reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}." + _output({"decision": "block", "reason": reason}) else: _output({}) @@ -320,6 +651,10 @@ def hook_precompact(data: dict, harness: str): _log(f"PRE-COMPACT triggered for session {session_id}") + # Capture tool output via our normalize path before compaction loses it + if transcript_path: + _ingest_transcript(transcript_path) + # Mine synchronously so data lands before compaction proceeds _mine_sync(transcript_path) diff --git a/mempalace/i18n/be.json b/mempalace/i18n/be.json new file mode 100644 index 000000000..01e808e85 --- /dev/null +++ b/mempalace/i18n/be.json @@ -0,0 +1,166 @@ +{ + "lang": "be", + "label": "Беларуская", + "terms": { + "palace": "палац", + "wing": "крыло", + "hall": "зала", + "closet": "шафа", + "drawer": "шуфляда", + "mine": "майнінг", + "search": "пошук", + "status": "статус", + "init": "ініцыялізацыя", + "repair": "аднаўленне", + "migrate": "міграцыя", + "entity": "аб'ект", + "topic": "тэма" + }, + "cli": { + "mine_start": "Майнінг {path}...", + "mine_complete": "Гатова. Створана шаф: {closets}, шуфляд: {drawers}.", + "mine_skip": "Майнінг скончаны. Выкарыстайце --force, каб перамайніць.", + "search_no_results": "Няма вынікаў для: {query}", + "search_results": "Знойдзена {count} вынікаў:", + "status_palace": "Палац: {path}", + "status_wings": "Крылаў: {count}", + "status_closets": "Шаф: {count}", + "status_drawers": "Шуфляд: {count}", + "init_complete": "Палац ініцыялізаваны ў {path}", + "init_exists": "Палац ужо існуе ў {path}", + "repair_complete": "Аднаўленне скончана. Выпраўлена праблем: {fixed}.", + "migrate_complete": "Міграцыя завершана.", + "no_palace": "Палац не знойдзены. Запусціце: mempalace init <дырэкторыя>" + }, + "aaak": { + "instruction": "Сцісні да фармату індэкса. Злучкі паміж словамі, вертыкальныя рысы паміж паняткамі. Выдалі падставы і службовыя словы. Захавай дакладнасць імёнаў, нумароў, скланенняў і формаў." + }, + "regex": { + "topic_pattern": "[А-ЯЁІЎ][а-яёіў]{2,}|[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_]{2,}", + "stop_words": "і а але бо каб калі калісьці хоць што хто дзе куды адкуль як чаму таму ж не ні так таксама яшчэ ўжо цяпер потым тут там адсюль туды ў на па з са без для над пад пры пра праз супраць замест акрамя сярод вакол уздоўж каля паміж пасля перад да ад у або альбо то ці быццам нібыта вось гэты гэта гэтыя той тая тыя некаторыя кожны кожная кожныя толькі вельмі можа трэба будзе быў была былі ёсць няма", + "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"", + "action_pattern": "(?:зрабіў|зрабіла|зрабілі|стварыў|стварыла|стварылі|дадаў|дадала|дадалі|абнавіў|абнавіла|абнавілі|наладзіў|наладзіла|наладзілі|пратэставаў|пратэставала|пратэставалі)\\s+[\\wа-яёА-ЯЁІіЎў\\s]{3,30}" + }, + "entity": { + "candidate_pattern": "[А-ЯЁІЎ][а-яёіў]{1,19}", + "multi_word_pattern": "[А-ЯЁІЎ][а-яёіў]+(?:\\s+[А-ЯЁІЎ][а-яёіў]+)+", + "person_verb_patterns": [ + "\\b{name}\\s+сказа(?:ў|ла|лі)\\b", + "\\b{name}\\s+спыта(?:ў|ла|лі)\\b", + "\\b{name}\\s+адказа(?:ў|ла|лі)\\b", + "\\b{name}\\s+расказа(?:ў|ла|лі)\\b", + "\\b{name}\\s+засмея(?:ўся|лася|ліся)\\b", + "\\b{name}\\s+усміхну(?:ўся|лася|ліся)\\b", + "\\b{name}\\s+заплака(?:ў|ла|лі)\\b", + "\\b{name}\\s+адчу(?:ў|ла|лі)\\b", + "\\b{name}\\s+думае\\b", + "\\b{name}\\s+хоча\\b", + "\\b{name}\\s+кахае\\b", + "\\b{name}\\s+ненавідзіць\\b", + "\\b{name}\\s+ведае\\b", + "\\b{name}\\s+вырашы(?:ў|ла|лі)\\b", + "\\b{name}\\s+напіса(?:ў|ла|лі)\\b", + "\\b{name}\\s+каза(?:ў|ла|лі)\\b", + "\\b{name}\\s+зрабі(?:ў|ла|лі)\\b", + "\\b{name}\\s+спытваецца\\b", + "\\b{name}\\s+адказвае\\b" + ], + "pronoun_patterns": [ + "\\bён\\b", + "\\bяго\\b", + "\\bяму\\b", + "\\bім\\b", + "\\bяна\\b", + "\\bяе\\b", + "\\bёй\\b", + "\\bёю\\b", + "\\bяны\\b", + "\\bіх\\b", + "\\bімі\\b", + "\\bяно\\b", + "\\bвы\\b", + "\\bвас\\b", + "\\bвам\\b", + "\\bвамі\\b" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:\\s]", + "^{name}:\\s", + "^\\[{name}\\]", + "\"{name}\\s+сказа(?:ў|ла|лі)" + ], + "direct_address_pattern": "\\bпрывітанне[,:!.]?\\s+{name}\\b|\\bвітаю[,:!.]?\\s+{name}\\b|\\bдзякуй[,:!.]?\\s+{name}\\b|\\bдарагі[,:!.]?\\s+{name}\\b|\\bдарагая[,:!.]?\\s+{name}\\b|\\bпаважаны[,:!.]?\\s+{name}\\b|\\bпаважаная[,:!.]?\\s+{name}\\b", + "project_verb_patterns": [ + "\\bзбіраю\\s+{name}\\b", + "\\bсабраў\\s+{name}\\b", + "\\bзапускаю\\s+{name}\\b", + "\\bзапусціў\\s+{name}\\b", + "\\bразгарнуў\\s+{name}\\b", + "\\bусталяваў\\s+{name}\\b", + "\\bсістэма\\s+{name}\\b", + "\\bпраект\\s+{name}\\b", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "і", + "ў", + "з", + "са", + "на", + "па", + "да", + "ад", + "у", + "без", + "для", + "над", + "пад", + "пры", + "пра", + "праз", + "супраць", + "замест", + "акрамя", + "сярод", + "вакол", + "уздоўж", + "каля", + "паміж", + "пасля", + "перад", + "або", + "альбо", + "каб", + "калі", + "хоць", + "бо", + "ж", + "не", + "ні", + "так", + "яшчэ", + "ужо", + "цяпер", + "вось", + "гэта", + "тое", + "тут", + "там", + "вельмі", + "толькі", + "можа", + "трэба", + "ёсць", + "няма", + "як", + "што", + "хто", + "чаму", + "таму", + "прывітанне", + "дзякуй", + "ласка" + ] + } +} diff --git a/mempalace/i18n/de.json b/mempalace/i18n/de.json index c6677b37d..f2476e119 100644 --- a/mempalace/i18n/de.json +++ b/mempalace/i18n/de.json @@ -40,5 +40,87 @@ "stop_words": "der die das ein eine eines einer einem einen den dem des und oder aber denn weil wenn als ob auch noch schon sehr viel nur nicht mehr kann wird hat ist sind war waren sein haben wurde mit von zu für auf in an um über nach durch", "quote_pattern": "\\u201E([^\\u201C]{10,200})\\u201C|\"([^\"]{10,200})\"", "action_pattern": "(?:gebaut|behoben|geschrieben|hinzugefügt|gepusht|gemessen|getestet|überprüft|erstellt|gelöscht|aktualisiert|konfiguriert|bereitgestellt|migriert)\\s+[\\wÄÖÜäöüß\\s]{3,30}" + }, + "entity": { + "candidate_pattern": "[A-ZÄÖÜ][a-zäöüß]{1,19}", + "multi_word_pattern": "[A-ZÄÖÜ][a-zäöüß]+(?:\\s+[A-ZÄÖÜ][a-zäöüß]+)+", + "person_verb_patterns": [ + "\\b{name}\\s+sagte\\b", + "\\b{name}\\s+fragte\\b", + "\\b{name}\\s+antwortete\\b", + "\\b{name}\\s+erzählte\\b", + "\\b{name}\\s+lachte\\b", + "\\b{name}\\s+lächelte\\b", + "\\b{name}\\s+weinte\\b", + "\\b{name}\\s+fühlte\\b", + "\\b{name}\\s+denkt\\b", + "\\b{name}\\s+will\\b", + "\\b{name}\\s+liebt\\b", + "\\b{name}\\s+hasst\\b", + "\\b{name}\\s+weiß\\b", + "\\b{name}\\s+entschied\\b", + "\\b{name}\\s+schrieb\\b" + ], + "pronoun_patterns": [ + "\\ber\\b", + "\\bsie\\b", + "\\bes\\b", + "\\bihn\\b", + "\\bihm\\b", + "\\bihr\\b", + "\\bsein\\b", + "\\bihre\\b", + "\\bihnen\\b" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:\\s]", + "^{name}:\\s", + "^\\[{name}\\]", + "\"{name}\\s+sagte" + ], + "direct_address_pattern": "\\bhallo\\s+{name}\\b|\\bhi\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bdanke\\s+{name}\\b|\\bservus\\s+{name}\\b|\\blieber\\s+{name}\\b|\\bliebe\\s+{name}\\b|\\bsehr\\s+geehrter\\s+{name}\\b|\\bsehr\\s+geehrte\\s+{name}\\b", + "project_verb_patterns": [ + "\\bbaue\\s+{name}\\b", + "\\bgebaut\\s+{name}\\b", + "\\bstarte\\s+{name}\\b", + "\\bgestartet\\s+{name}\\b", + "\\bdeploye\\s+{name}\\b", + "\\binstalliert\\s+{name}\\b", + "\\bdie\\s+{name}\\s+architektur\\b", + "\\bdie\\s+{name}\\s+pipeline\\b", + "\\bdas\\s+{name}\\s+system\\b", + "\\bdas\\s+{name}\\s+repository\\b", + "\\b{name}\\s+v\\d+\\b", + "\\b{name}\\.py\\b", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "der", "die", "das", "ein", "eine", "eines", "einer", "einem", "einen", + "den", "dem", "des", "und", "oder", "aber", "denn", "weil", "wenn", "als", + "ob", "auch", "noch", "schon", "sehr", "viel", "nur", "nicht", "mehr", + "kann", "wird", "hat", "ist", "sind", "war", "waren", "sein", "haben", + "wurde", "worden", "werden", "mit", "von", "zu", "für", "auf", "in", + "an", "um", "über", "nach", "durch", "bei", "aus", "seit", "vor", "zwischen", + "ich", "du", "er", "sie", "es", "wir", "ihr", "mich", "dich", "mir", "dir", + "uns", "euch", "mein", "dein", "unser", "euer", "ihre", "seine", + "wer", "was", "wann", "wo", "wie", "warum", "welcher", "welche", "welches", + "so", "dann", "jetzt", "heute", "gestern", "morgen", "hier", "dort", "da", + "immer", "nie", "manchmal", "oft", "selten", "bald", "spät", + "ja", "nein", "vielleicht", "gut", "schlecht", "besser", "bitte", "danke", + "hallo", "hi", "hey", "tschüss", + "tag", "tage", "woche", "monat", "jahr", "jahre", "zeit", "welt", "leben", + "mensch", "menschen", "leute", "person", "ding", "dinge", "sache", "sachen", + "teil", "art", "weise", "stelle", "platz", "ort", "zimmer", "haus", "land", + "grund", "frage", "antwort", "fakt", "sinn", "idee", "punkt", "fall", "aspekt", + "beispiel", "version", "nummer", "zahl", "name", "namen", "system", "modell", + "sprache", "technologie", "gesellschaft", "kultur", "geschichte", + "wissenschaft", "zukunft", "erinnerung", "gedächtnis", + "datei", "ordner", "pfad", "schlüssel", "wert", "fehler", "warnung", + "ergebnis", "eingabe", "ausgabe", "quelle", "ziel", "daten", "elemente", + "montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag", "sonntag", + "januar", "februar", "märz", "april", "mai", "juni", "juli", "august", + "september", "oktober", "november", "dezember" + ] } } diff --git a/mempalace/i18n/en.json b/mempalace/i18n/en.json index 6a9dff925..39d9ac140 100644 --- a/mempalace/i18n/en.json +++ b/mempalace/i18n/en.json @@ -42,7 +42,7 @@ "action_pattern": "(?:built|fixed|wrote|added|pushed|measured|tested|reviewed|created|deleted|updated|configured|deployed|migrated)\\s+[\\w\\s]{3,30}" }, "entity": { - "candidate_pattern": "[A-Z][a-z]{1,19}", + "candidate_pattern": "[A-Z][a-z]+(?:[A-Z][a-z]+|[A-Z]{2,})+|[A-Z][a-z]{1,19}", "multi_word_pattern": "[A-Z][a-z]+(?:\\s+[A-Z][a-z]+)+", "person_verb_patterns": [ "\\b{name}\\s+said\\b", @@ -140,7 +140,17 @@ "agents", "tools", "others", "guards", "ethics", "regulation", "learning", "thinking", "memory", "language", "intelligence", "technology", "society", "culture", "future", "history", "science", - "model", "models", "network", "networks", "training", "inference" + "model", "models", "network", "networks", "training", "inference", + "created", "updated", "deleted", "added", "removed", "modified", + "extracted", "processed", "generated", "compiled", "launched", "installed", + "deployed", "executed", "loaded", "parsed", "validated", "configured", + "total", "summary", "covered", "included", "pending", "failed", "success", + "ready", "active", "disabled", "enabled", "available", "completed", + "auto", "multi", "mini", "micro", "meta", "super", "hybrid", + "context", "bridge", "batch", "local", "global", "native", "cloud", + "before", "after", "during", "often", "always", "never", + "project", "contributor", "software", + "backend", "frontend", "server", "client", "service", "app", "api" ] } } diff --git a/mempalace/i18n/es.json b/mempalace/i18n/es.json index aa30e1bcb..dd490aab9 100644 --- a/mempalace/i18n/es.json +++ b/mempalace/i18n/es.json @@ -40,5 +40,89 @@ "stop_words": "el la los las un una unos unas de del al en con por para su sus mi mis tu tus es son está están fue ser estar haber sido como pero más muy también todo todos toda todas este esta estos estas ese esa esos esas que quien cual donde cuando porque aunque sin", "quote_pattern": "\"([^\"]{10,200})\"|«([^»]{10,200})»", "action_pattern": "(?:construido|corregido|escrito|añadido|enviado|medido|probado|revisado|creado|eliminado|actualizado|configurado|desplegado|migrado)\\s+[\\wá-ú\\s]{3,30}" + }, + "entity": { + "candidate_pattern": "[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]{1,19}", + "multi_word_pattern": "[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]+(?:\\s+[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]+)+", + "person_verb_patterns": [ + "\\b{name}\\s+dijo\\b", + "\\b{name}\\s+preguntó\\b", + "\\b{name}\\s+respondió\\b", + "\\b{name}\\s+contó\\b", + "\\b{name}\\s+rió\\b", + "\\b{name}\\s+sonrió\\b", + "\\b{name}\\s+lloró\\b", + "\\b{name}\\s+sintió\\b", + "\\b{name}\\s+piensa\\b", + "\\b{name}\\s+quiere\\b", + "\\b{name}\\s+ama\\b", + "\\b{name}\\s+odia\\b", + "\\b{name}\\s+sabe\\b", + "\\b{name}\\s+decidió\\b", + "\\b{name}\\s+escribió\\b" + ], + "pronoun_patterns": [ + "\\bél\\b", + "\\bella\\b", + "\\bellos\\b", + "\\bellas\\b", + "\\blo\\b", + "\\bla\\b", + "\\ble\\b", + "\\bles\\b", + "\\bse\\b" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:\\s]", + "^{name}:\\s", + "^\\[{name}\\]", + "\"{name}\\s+dijo" + ], + "direct_address_pattern": "\\bhola\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bhi\\s+{name}\\b|\\bgracias\\s+{name}\\b|\\bquerido\\s+{name}\\b|\\bquerida\\s+{name}\\b|\\bestimado\\s+{name}\\b|\\bestimada\\s+{name}\\b|\\bdon\\s+{name}\\b|\\bdoña\\s+{name}\\b|\\bseñor\\s+{name}\\b|\\bseñora\\s+{name}\\b", + "project_verb_patterns": [ + "\\bconstruyo\\s+{name}\\b", + "\\bconstruí\\s+{name}\\b", + "\\barmé\\s+{name}\\b", + "\\blancé\\s+{name}\\b", + "\\bdesplegué\\s+{name}\\b", + "\\binstalé\\s+{name}\\b", + "\\bla\\s+arquitectura\\s+{name}\\b", + "\\bel\\s+pipeline\\s+{name}\\b", + "\\bel\\s+sistema\\s+{name}\\b", + "\\bel\\s+proyecto\\s+{name}\\b", + "\\bel\\s+repositorio\\s+{name}\\b", + "\\b{name}\\s+v\\d+\\b", + "\\b{name}\\.py\\b", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "el", "la", "los", "las", "un", "una", "unos", "unas", + "de", "del", "al", "a", "en", "con", "sin", "por", "para", "sobre", + "entre", "hasta", "desde", "hacia", "contra", "según", "tras", + "y", "o", "u", "ni", "pero", "sino", "aunque", "porque", "pues", + "que", "quien", "quienes", "cual", "cuales", "cuyo", "cuya", + "donde", "cuando", "como", "cuanto", "cuanta", + "yo", "tú", "él", "ella", "nosotros", "vosotros", "ellos", "ellas", + "me", "te", "se", "nos", "os", "lo", "la", "le", "los", "las", "les", + "mi", "mis", "tu", "tus", "su", "sus", "nuestro", "nuestra", "vuestro", + "este", "esta", "estos", "estas", "ese", "esa", "esos", "esas", + "aquel", "aquella", "aquellos", "aquellas", "esto", "eso", "aquello", + "ser", "estar", "haber", "tener", "hacer", "poder", "querer", "saber", + "es", "son", "fue", "fueron", "era", "eran", "está", "están", "estaba", + "he", "ha", "hemos", "han", "había", "hay", + "muy", "mucho", "mucha", "muchos", "muchas", "poco", "poca", "pocos", "pocas", + "más", "menos", "tan", "tanto", "también", "tampoco", + "sí", "no", "quizás", "tal", "vez", + "aquí", "allí", "allá", "ahí", "acá", + "hoy", "ayer", "mañana", "ahora", "antes", "después", "luego", "entonces", + "siempre", "nunca", "jamás", "todavía", "aún", "ya", + "bien", "mal", "mejor", "peor", "bueno", "buena", "malo", "mala", + "gracias", "hola", "adiós", "por favor", "perdón", + "día", "días", "semana", "mes", "año", "años", "tiempo", "vez", "veces", + "cosa", "cosas", "persona", "gente", "mundo", "vida", "casa", "lugar", + "forma", "manera", "parte", "caso", "punto", "idea", "hecho", "razón", + "nombre", "número", "versión", "sistema", "modelo" + ] } } diff --git a/mempalace/i18n/fr.json b/mempalace/i18n/fr.json index 2e3d0b9e3..86df08c72 100644 --- a/mempalace/i18n/fr.json +++ b/mempalace/i18n/fr.json @@ -40,5 +40,87 @@ "stop_words": "le la les un une des de du au aux en et ou mais donc or ni car que qui ce cette ces son sa ses mon ma mes ton ta tes leur leurs nous vous ils elles on ne pas plus très bien aussi avec pour dans sur par est sont fait être avoir été comme tout tous toute toutes", "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"", "action_pattern": "(?:construit|corrigé|écrit|ajouté|poussé|mesuré|testé|révisé|créé|supprimé|mis à jour|configuré|déployé|migré)\\s+[\\wà-ÿ\\s]{3,30}" + }, + "entity": { + "candidate_pattern": "[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]{1,19}", + "multi_word_pattern": "[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]+(?:\\s+[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]+)+", + "person_verb_patterns": [ + "\\b{name}\\s+a\\s+dit\\b", + "\\b{name}\\s+a\\s+demandé\\b", + "\\b{name}\\s+a\\s+répondu\\b", + "\\b{name}\\s+a\\s+raconté\\b", + "\\b{name}\\s+a\\s+ri\\b", + "\\b{name}\\s+a\\s+souri\\b", + "\\b{name}\\s+a\\s+pleuré\\b", + "\\b{name}\\s+a\\s+senti\\b", + "\\b{name}\\s+pense\\b", + "\\b{name}\\s+veut\\b", + "\\b{name}\\s+aime\\b", + "\\b{name}\\s+déteste\\b", + "\\b{name}\\s+sait\\b", + "\\b{name}\\s+a\\s+décidé\\b", + "\\b{name}\\s+a\\s+écrit\\b" + ], + "pronoun_patterns": [ + "\\bil\\b", + "\\belle\\b", + "\\blui\\b", + "\\bils\\b", + "\\belles\\b", + "\\bleur\\b", + "\\bleurs\\b", + "\\beux\\b", + "\\bse\\b" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:\\s]", + "^{name}:\\s", + "^\\[{name}\\]", + "\"{name}\\s+a\\s+dit" + ], + "direct_address_pattern": "\\bbonjour\\s+{name}\\b|\\bsalut\\s+{name}\\b|\\bmerci\\s+{name}\\b|\\bcher\\s+{name}\\b|\\bchère\\s+{name}\\b|\\bmonsieur\\s+{name}\\b|\\bmadame\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bhi\\s+{name}\\b", + "project_verb_patterns": [ + "\\bconstruit\\s+{name}\\b", + "\\blancé\\s+{name}\\b", + "\\bdéployé\\s+{name}\\b", + "\\binstallé\\s+{name}\\b", + "\\bl'architecture\\s+{name}\\b", + "\\ble\\s+pipeline\\s+{name}\\b", + "\\ble\\s+système\\s+{name}\\b", + "\\ble\\s+projet\\s+{name}\\b", + "\\ble\\s+dépôt\\s+{name}\\b", + "\\b{name}\\s+v\\d+\\b", + "\\b{name}\\.py\\b", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "le", "la", "les", "un", "une", "des", "du", "de", "au", "aux", + "en", "dans", "sur", "sous", "avec", "sans", "pour", "par", "vers", + "chez", "entre", "depuis", "pendant", "avant", "après", "jusqu", + "et", "ou", "mais", "donc", "or", "ni", "car", "que", "qui", + "dont", "où", "quand", "comment", "pourquoi", "combien", "lequel", + "ce", "cet", "cette", "ces", "celui", "celle", "ceux", "celles", + "mon", "ma", "mes", "ton", "ta", "tes", "son", "sa", "ses", + "notre", "nos", "votre", "vos", "leur", "leurs", + "je", "tu", "il", "elle", "on", "nous", "vous", "ils", "elles", + "me", "te", "se", "lui", "eux", + "être", "avoir", "faire", "dire", "aller", "voir", "savoir", "pouvoir", + "est", "sont", "était", "étaient", "fut", "furent", "sera", "seront", + "ai", "as", "a", "avons", "avez", "ont", "avait", "avaient", + "très", "bien", "mal", "peu", "beaucoup", "trop", "assez", "aussi", + "plus", "moins", "tant", "si", "tellement", + "oui", "non", "peut-être", "vraiment", + "ici", "là", "là-bas", "partout", "ailleurs", + "aujourd'hui", "hier", "demain", "maintenant", "alors", "ensuite", + "toujours", "jamais", "souvent", "parfois", "déjà", "encore", + "bon", "bonne", "mauvais", "mauvaise", "meilleur", "pire", + "merci", "bonjour", "salut", "au revoir", + "jour", "jours", "semaine", "mois", "année", "temps", "fois", + "chose", "choses", "personne", "gens", "monde", "vie", "maison", + "endroit", "lieu", "partie", "façon", "manière", "sorte", "type", + "cas", "point", "idée", "fait", "raison", "nom", "nombre", + "version", "système", "modèle", "question", "réponse" + ] } } diff --git a/mempalace/i18n/zh-CN.json b/mempalace/i18n/zh-CN.json index 4e41a5714..7a708cf0a 100644 --- a/mempalace/i18n/zh-CN.json +++ b/mempalace/i18n/zh-CN.json @@ -40,5 +40,93 @@ "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一个 上 也 很 到 说 要 去 你 会 着 没有 看 好 自己 这 那 她 他 它 们 但是 因为 所以 如果 虽然 然后 或者 而且", "quote_pattern": "\\u201C([^\\u201D]{10,100})\\u201D|\"([^\"]{10,200})\"", "action_pattern": "(构建|修复|添加|删除|确认|创建|实现|修理|编写|测试|验证|更新|配置|启动|停止)(?:了|完成|成功)" + }, + "entity": { + "boundary_chars": "\\u4E00-\\u9FFF", + "candidate_pattern": "[王李张刘陈杨赵黄周吴徐孙朱胡郭何高林罗郑梁谢宋唐许韩冯邓曹彭曾萧田董袁潘于蒋蔡余杜叶程苏魏吕丁任沈姚卢姜崔钟谭陆汪范金石廖贾夏韦方白邹孟熊秦邱江尹薛阎段雷侯龙史陶黎贺顾毛郝龚邵万钱严武戴莫孔向汤温庞殷章葛管甘卞冉蓝殷习][\\u4E00-\\u9FFF]{1,2}", + "person_verb_patterns": [ + "{name}说", + "{name}问", + "{name}答", + "{name}表示", + "{name}回答", + "{name}提出", + "{name}决定", + "{name}认为", + "{name}指出", + "{name}解释", + "{name}告诉", + "{name}写道", + "{name}想", + "{name}觉得", + "{name}知道", + "{name}喜欢", + "{name}讨厌", + "{name}确认", + "{name}提醒", + "{name}分享", + "{name}建议", + "{name}同意", + "{name}反对" + ], + "pronoun_patterns": [ + "他们", + "她们", + "他", + "她", + "它", + "您", + "咱" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:：\\s]", + "^{name}[:：]\\s?", + "^\\[{name}\\]", + "\u201C{name}[\u201D:：]", + "「{name}[」:：]" + ], + "direct_address_pattern": "嘿\\s*{name}|喂\\s*{name}|谢谢\\s*{name}|感谢\\s*{name}|哈喽\\s*{name}|亲爱的\\s*{name}", + "project_verb_patterns": [ + "建立{name}", + "打造{name}", + "部署{name}", + "启动{name}", + "发布{name}", + "上线{name}", + "开发{name}", + "维护{name}", + "{name}系统", + "{name}平台", + "{name}项目", + "{name}架构", + "{name}管线", + "{name}v\\d+", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "的", "了", "着", "过", "得", "地", "吗", "吧", "呢", "啊", "喔", "耶", + "我", "你", "妳", "他", "她", "它", "您", "咱", + "我们", "你们", "妳们", "他们", "她们", "它们", "咱们", + "自己", "大家", "有人", "没人", + "今天", "明天", "昨天", "前天", "后天", "今年", "明年", "去年", + "早上", "下午", "晚上", "中午", "凌晨", + "现在", "刚才", "刚刚", "等等", "等下", "待会", + "最近", "以前", "之前", "之后", "以后", "后来", + "什么", "为什么", "怎么", "怎样", "哪里", "哪个", + "这个", "那个", "这里", "那里", "这些", "那些", "这样", "那样", + "但是", "可是", "然后", "所以", "因为", "如果", "虽然", + "而且", "或者", "或是", "还是", "不过", "只是", "不只", + "既然", "不然", "否则", "此外", "另外", + "很", "非常", "相当", "真的", "确实", "当然", "其实", + "已经", "正在", "即将", "将要", "刚好", "恰好", + "可能", "也许", "或许", "大概", "应该", "必须", "一定", + "完成", "执行", "进行", "开始", "结束", "继续", "停止", "完毕", + "没有", "有点", "有些", "一些", "许多", "很多", + "问题", "答案", "原因", "结果", "情况", "状况", + "主要", "重要", "基本", "简单", "复杂", "特别", + "谢谢", "感谢", "对不起", "不好意思", "请问", + "欢迎", "再见", "你好", "您好", "哈喽", "拜拜" + ] } } diff --git a/mempalace/i18n/zh-TW.json b/mempalace/i18n/zh-TW.json index b65552bce..db3f2ad0d 100644 --- a/mempalace/i18n/zh-TW.json +++ b/mempalace/i18n/zh-TW.json @@ -40,5 +40,93 @@ "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一個 上 也 很 到 說 要 去 你 會 著 沒有 看 好 自己 這 那 她 他 它 們 但是 因為 所以 如果 雖然 然後 或者 而且", "quote_pattern": "「([^」]{10,100})」|\u201c([^\u201d]{10,100})\u201d", "action_pattern": "(構建|修復|添加|刪除|確認|創建|實現|修理|編寫|測試|驗證|更新|配置|啟動|停止)(?:了|完成|成功)" + }, + "entity": { + "boundary_chars": "\\u4E00-\\u9FFF", + "candidate_pattern": "[王李張劉陳楊趙黃周吳徐孫朱胡郭何高林羅鄭梁謝宋唐許韓馮鄧曹彭曾蕭田董袁潘于蔣蔡余杜葉程蘇魏呂丁任沈姚盧姜崔鍾譚陸汪范金石廖賈夏韋方白鄒孟熊秦邱江尹薛閻段雷侯龍史陶黎賀顧毛郝龔邵萬錢嚴武戴莫孔向湯溫龐殷章葛管甘卞冉藍殷習][\\u4E00-\\u9FFF]{1,2}", + "person_verb_patterns": [ + "{name}說", + "{name}問", + "{name}答", + "{name}表示", + "{name}回答", + "{name}提出", + "{name}決定", + "{name}認為", + "{name}指出", + "{name}解釋", + "{name}告訴", + "{name}寫道", + "{name}想", + "{name}覺得", + "{name}知道", + "{name}喜歡", + "{name}討厭", + "{name}確認", + "{name}提醒", + "{name}分享", + "{name}建議", + "{name}同意", + "{name}反對" + ], + "pronoun_patterns": [ + "他們", + "她們", + "他", + "她", + "它", + "您", + "咱" + ], + "dialogue_patterns": [ + "^>\\s*{name}[:：\\s]", + "^{name}[:：]\\s?", + "^\\[{name}\\]", + "「{name}[」:：]", + "『{name}[』:：]" + ], + "direct_address_pattern": "嘿\\s*{name}|喂\\s*{name}|謝謝\\s*{name}|感謝\\s*{name}|哈囉\\s*{name}|親愛的\\s*{name}", + "project_verb_patterns": [ + "建立{name}", + "打造{name}", + "部署{name}", + "啟動{name}", + "發布{name}", + "上線{name}", + "開發{name}", + "維護{name}", + "{name}系統", + "{name}平台", + "{name}專案", + "{name}架構", + "{name}管線", + "{name}v\\d+", + "\\bimport\\s+{name}\\b", + "\\bpip\\s+install\\s+{name}\\b" + ], + "stopwords": [ + "的", "了", "著", "過", "得", "地", "嗎", "吧", "呢", "啊", "喔", "耶", + "我", "你", "妳", "他", "她", "它", "您", "咱", + "我們", "你們", "妳們", "他們", "她們", "它們", "咱們", + "自己", "大家", "有人", "沒人", + "今天", "明天", "昨天", "前天", "後天", "今年", "明年", "去年", + "早上", "下午", "晚上", "中午", "凌晨", + "現在", "剛才", "剛剛", "等等", "等下", "待會", + "最近", "以前", "之前", "之後", "以後", "後來", + "什麼", "甚麼", "為什麼", "怎麼", "怎樣", "哪裡", "哪個", + "這個", "那個", "這裡", "那裡", "這些", "那些", "這樣", "那樣", + "但是", "可是", "然後", "所以", "因為", "如果", "雖然", + "而且", "或者", "或是", "還是", "不過", "只是", "不只", + "既然", "不然", "否則", "此外", "另外", + "很", "非常", "相當", "真的", "確實", "當然", "其實", + "已經", "正在", "即將", "將要", "剛好", "恰好", + "可能", "也許", "或許", "大概", "應該", "必須", "一定", + "完成", "執行", "進行", "開始", "結束", "繼續", "停止", "完畢", + "沒有", "有點", "有些", "一些", "許多", "很多", + "問題", "答案", "原因", "結果", "情況", "狀況", + "主要", "重要", "基本", "簡單", "複雜", "特別", + "謝謝", "感謝", "對不起", "不好意思", "請問", + "歡迎", "再見", "你好", "您好", "哈囉", "掰掰" + ] } } diff --git a/mempalace/instructions/init.md b/mempalace/instructions/init.md index 40f0c20dd..570a52541 100644 --- a/mempalace/instructions/init.md +++ b/mempalace/instructions/init.md @@ -49,7 +49,7 @@ If this fails, report the error and stop. Run the following command to register the MemPalace MCP server with Claude: - claude mcp add mempalace -- python -m mempalace.mcp_server + claude mcp add mempalace -- mempalace-mcp If this fails, report the error but continue to the next step (MCP configuration can be done manually later). diff --git a/mempalace/llm_client.py b/mempalace/llm_client.py new file mode 100644 index 000000000..74982cea1 --- /dev/null +++ b/mempalace/llm_client.py @@ -0,0 +1,305 @@ +""" +llm_client.py — Minimal provider abstraction for LLM-assisted entity refinement. + +Three providers cover the useful space: + +- ``ollama`` (default): local models via http://localhost:11434. Works fully + offline. Honors MemPalace's "zero-API required" principle. +- ``openai-compat``: any OpenAI-compatible ``/v1/chat/completions`` endpoint. + Covers OpenRouter, LM Studio, llama.cpp server, vLLM, Groq, Fireworks, + Together, and most self-hosted setups. +- ``anthropic``: the official Messages API. Opt-in for users who want Haiku + quality without setting up a local model. + +All providers expose the same ``classify(system, user, json_mode)`` method and +the same ``check_available()`` probe. No external SDK dependencies — stdlib +``urllib`` only. + +JSON mode matters here: we always ask for structured output. Providers +differ on how to request it (Ollama: ``format: json``; OpenAI-compat: +``response_format``; Anthropic: prompt-level instruction) and this module +normalizes that away from the caller. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from typing import Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +class LLMError(RuntimeError): + """Raised for any provider failure — transport, parse, auth, missing model.""" + + +@dataclass +class LLMResponse: + text: str + model: str + provider: str + raw: dict + + +# ==================== BASE ==================== + + +class LLMProvider: + name: str = "base" + + def __init__( + self, + model: str, + endpoint: Optional[str] = None, + api_key: Optional[str] = None, + timeout: int = 120, + ): + self.model = model + self.endpoint = endpoint + self.api_key = api_key + self.timeout = timeout + + def classify(self, system: str, user: str, json_mode: bool = True) -> LLMResponse: + raise NotImplementedError + + def check_available(self) -> tuple[bool, str]: + """Return ``(ok, message)``. Fast probe that the provider is reachable.""" + raise NotImplementedError + + +def _http_post_json(url: str, body: dict, headers: dict, timeout: int) -> dict: + """POST JSON and return the parsed response. Raises LLMError on any failure.""" + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", **headers}, + ) + try: + with urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + except HTTPError as e: + detail = "" + try: + detail = e.read().decode("utf-8", errors="replace")[:500] + except Exception: + pass + raise LLMError(f"HTTP {e.code} from {url}: {detail or e.reason}") from e + except (URLError, OSError) as e: + raise LLMError(f"Cannot reach {url}: {e}") from e + except json.JSONDecodeError as e: + raise LLMError(f"Malformed response from {url}: {e}") from e + + +# ==================== OLLAMA ==================== + + +class OllamaProvider(LLMProvider): + name = "ollama" + DEFAULT_ENDPOINT = "http://localhost:11434" + + def __init__( + self, + model: str, + endpoint: Optional[str] = None, + timeout: int = 180, + **_: object, + ): + super().__init__( + model=model, + endpoint=endpoint or self.DEFAULT_ENDPOINT, + timeout=timeout, + ) + + def check_available(self) -> tuple[bool, str]: + try: + with urlopen(f"{self.endpoint}/api/tags", timeout=5) as resp: + data = json.loads(resp.read()) + except (URLError, HTTPError, OSError, json.JSONDecodeError) as e: + return False, f"Cannot reach Ollama at {self.endpoint}: {e}" + names = {m.get("name", "") for m in data.get("models", []) or []} + # Ollama tags may or may not include ':latest' — accept either form + wanted = {self.model, f"{self.model}:latest"} + if not names & wanted: + return ( + False, + f"Model '{self.model}' not loaded in Ollama. Run: ollama pull {self.model}", + ) + return True, "ok" + + def classify(self, system: str, user: str, json_mode: bool = True) -> LLMResponse: + body: dict = { + "model": self.model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "stream": False, + "options": {"temperature": 0.1}, + } + if json_mode: + body["format"] = "json" + data = _http_post_json(f"{self.endpoint}/api/chat", body, headers={}, timeout=self.timeout) + text = (data.get("message") or {}).get("content", "") + if not text: + raise LLMError(f"Empty response from Ollama (model={self.model})") + return LLMResponse(text=text, model=self.model, provider=self.name, raw=data) + + +# ==================== OPENAI-COMPAT ==================== + + +class OpenAICompatProvider(LLMProvider): + """Any OpenAI-compatible ``/v1/chat/completions`` endpoint. + + Supply ``--llm-endpoint http://host:port`` (with or without ``/v1``). + API key via ``--llm-api-key`` or the ``OPENAI_API_KEY`` env var. + """ + + name = "openai-compat" + + def __init__( + self, + model: str, + endpoint: Optional[str] = None, + api_key: Optional[str] = None, + timeout: int = 120, + **_: object, + ): + resolved_key = api_key or os.environ.get("OPENAI_API_KEY") + super().__init__(model=model, endpoint=endpoint, api_key=resolved_key, timeout=timeout) + + def _resolve_url(self) -> str: + if not self.endpoint: + raise LLMError("openai-compat provider requires --llm-endpoint") + url = self.endpoint.rstrip("/") + if url.endswith("/chat/completions"): + return url + if not url.endswith("/v1"): + url = f"{url}/v1" + return f"{url}/chat/completions" + + def check_available(self) -> tuple[bool, str]: + if not self.endpoint: + return False, "no --llm-endpoint configured" + base = self.endpoint.rstrip("/") + base = base.removesuffix("/chat/completions").removesuffix("/v1") + try: + req = Request(f"{base}/v1/models") + if self.api_key: + req.add_header("Authorization", f"Bearer {self.api_key}") + with urlopen(req, timeout=5): + pass + except (URLError, HTTPError, OSError) as e: + return False, f"Cannot reach {self.endpoint}: {e}" + return True, "ok" + + def classify(self, system: str, user: str, json_mode: bool = True) -> LLMResponse: + body: dict = { + "model": self.model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "temperature": 0.1, + } + if json_mode: + body["response_format"] = {"type": "json_object"} + headers = {} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + data = _http_post_json(self._resolve_url(), body, headers=headers, timeout=self.timeout) + try: + text = data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as e: + raise LLMError(f"Unexpected response shape: {e}") from e + if not text: + raise LLMError(f"Empty response from {self.name} (model={self.model})") + return LLMResponse(text=text, model=self.model, provider=self.name, raw=data) + + +# ==================== ANTHROPIC ==================== + + +class AnthropicProvider(LLMProvider): + name = "anthropic" + DEFAULT_ENDPOINT = "https://api.anthropic.com" + API_VERSION = "2023-06-01" + + def __init__( + self, + model: str, + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + timeout: int = 120, + **_: object, + ): + key = api_key or os.environ.get("ANTHROPIC_API_KEY") + super().__init__( + model=model, + endpoint=endpoint or self.DEFAULT_ENDPOINT, + api_key=key, + timeout=timeout, + ) + + def check_available(self) -> tuple[bool, str]: + if not self.api_key: + return False, "ANTHROPIC_API_KEY not set (use --llm-api-key or env)" + # Don't probe — a live request would cost money. First real call will + # surface auth errors if the key is invalid. + return True, "ok" + + def classify(self, system: str, user: str, json_mode: bool = True) -> LLMResponse: + if not self.api_key: + raise LLMError("Anthropic provider requires ANTHROPIC_API_KEY env or --llm-api-key") + sys_prompt = system + if json_mode: + sys_prompt += "\n\nRespond with valid JSON only, no prose." + body = { + "model": self.model, + "max_tokens": 2048, + "temperature": 0.1, + "system": sys_prompt, + "messages": [{"role": "user", "content": user}], + } + headers = { + "X-API-Key": self.api_key, + "anthropic-version": self.API_VERSION, + } + data = _http_post_json( + f"{self.endpoint}/v1/messages", body, headers=headers, timeout=self.timeout + ) + try: + text = "".join( + b.get("text", "") for b in data.get("content", []) or [] if b.get("type") == "text" + ) + except (AttributeError, TypeError) as e: + raise LLMError(f"Unexpected response shape: {e}") from e + if not text: + raise LLMError(f"Empty response from Anthropic (model={self.model})") + return LLMResponse(text=text, model=self.model, provider=self.name, raw=data) + + +# ==================== FACTORY ==================== + + +PROVIDERS: dict[str, type[LLMProvider]] = { + "ollama": OllamaProvider, + "openai-compat": OpenAICompatProvider, + "anthropic": AnthropicProvider, +} + + +def get_provider( + name: str, + model: str, + endpoint: Optional[str] = None, + api_key: Optional[str] = None, + timeout: int = 120, +) -> LLMProvider: + """Build a provider by name. Raises LLMError on unknown provider.""" + cls = PROVIDERS.get(name) + if cls is None: + raise LLMError(f"Unknown provider '{name}'. Choices: {sorted(PROVIDERS.keys())}") + return cls(model=model, endpoint=endpoint, api_key=api_key, timeout=timeout) diff --git a/mempalace/llm_refine.py b/mempalace/llm_refine.py new file mode 100644 index 000000000..faa737ae4 --- /dev/null +++ b/mempalace/llm_refine.py @@ -0,0 +1,446 @@ +""" +llm_refine.py — Optional LLM refinement of regex-detected entities. + +Takes the candidate set produced by phase-1 detection (manifests, git +authors, regex on prose) and asks an LLM to reclassify each candidate as +PERSON / PROJECT / TOPIC / COMMON_WORD / AMBIGUOUS. + +Design constraints: +- Opt-in. Default init path never imports this module. +- Local-first by default (Ollama). +- Interactive UX: visible progress, clean cancellation (Ctrl-C returns + whatever was classified before the interrupt). +- Don't feed the raw corpus to the LLM — feed candidates + a few sampled + context lines each. Keeps total input to ~50-100K tokens even for huge + prose corpora. + +Public: + refine_entities(detected, corpus_text, provider, ...) -> dict +""" + +from __future__ import annotations + +import json +import re +import sys +from dataclasses import dataclass + +from mempalace.llm_client import LLMError, LLMProvider + + +BATCH_SIZE = 25 # candidates per LLM call; tuned for 4B local models +CONTEXT_LINES_PER_CANDIDATE = 3 +CONTEXT_WINDOW_CHARS = 240 # max chars per context line to keep tokens bounded + +# Valid labels the LLM is allowed to return. Anything else is treated as +# AMBIGUOUS so the user reviews it. +VALID_LABELS = {"PERSON", "PROJECT", "TOPIC", "COMMON_WORD", "AMBIGUOUS"} + + +SYSTEM_PROMPT = """You are helping organize a user's memory palace by classifying capitalized tokens found in their files. + +For each candidate, pick exactly ONE label: +- PERSON: a specific real person the user knows (colleague, family, character they write about) +- PROJECT: a named product, codebase, or effort the user works on +- TOPIC: a recurring theme or subject (not a person, not a project) — cities, technologies, concepts +- COMMON_WORD: an English word, verb, or fragment that isn't a named entity at all (e.g. "Created", "Before", "Never") +- AMBIGUOUS: context is insufficient to decide between two of the above + +Frameworks, runtimes, APIs, cloud services, vendors, and third-party products +(e.g. Angular, OpenAPI, Terraform, Bun, Google) are TOPIC unless the context +clearly says this is the user's own named codebase, product, or active effort. + +Use the provided context lines to disambiguate. A capitalized word that only appears in metadata ("Created: 2026-04-24") is COMMON_WORD. A name that appears with pronouns and dialogue is PERSON. + +Respond with JSON only. Schema: +{"classifications": [{"name": "", "label": "