feat: add sessionless local tool attribution flow (#23)

* docs(specs): add sessionless local tool attribution design

* feat(backend): add initial tool usage event attribution path

* feat(backend): add tool usage handler coverage

* feat(ae-cli): add local attribution parser and sync skeleton

* feat(ae-cli): trigger attribution sync from git hooks

* feat(attribution): bind local tool usage to checkpoints

* feat(session): finalize sessionless local attribution flow

* fix(review): tighten workspace scope and parser behavior

* fix(sync): prune replayed spool events incrementally

Author: LichKing-2234

ae-cli/cmd/hook.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/ai-efficiency/ae-cli/internal/attributionlocal"
 	"github.com/ai-efficiency/ae-cli/internal/hooks"
 	"github.com/ai-efficiency/ae-cli/internal/proxy"
 	"github.com/spf13/cobra"
@@ -72,10 +73,22 @@ var hookSessionEventCmd = &cobra.Command{
 	},
 }
 
+var hookAttributionSyncCmd = &cobra.Command{
+	Use:    "attribution-sync",
+	Short:  "Run local attribution sync (hidden)",
+	Hidden: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cwd, _ := os.Getwd()
+		engine := attributionlocal.NewSyncEngine(apiClient)
+		return engine.RunForWorkspace(context.Background(), cwd)
+	},
+}
+
 func init() {
 	hookCmd.AddCommand(hookPostCommitCmd)
 	hookCmd.AddCommand(hookPostRewriteCmd)
 	hookSessionEventCmd.Flags().String("tool", "", "originating tool name")
 	hookCmd.AddCommand(hookSessionEventCmd)
+	hookCmd.AddCommand(hookAttributionSyncCmd)
 	rootCmd.AddCommand(hookCmd)
 }

ae-cli/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
+	golang.org/x/sys v0.38.0
 )
 
 require (
@@ -23,8 +24,10 @@ require (
 	github.com/clipperhouse/displaywidth v0.9.0 // indirect
 	github.com/clipperhouse/stringish v0.1.1 // indirect
 	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/glebarez/go-sqlite v1.22.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
@@ -37,6 +40,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -49,8 +53,11 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.37.6 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.28.0 // indirect
 )

ae-cli/go.sum

@@ -27,12 +27,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -68,6 +72,8 @@ github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@@ -123,3 +129,11 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
+modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
+modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=

ae-cli/internal/attributionlocal/claude_jsonl.go

@@ -0,0 +1,88 @@
+package attributionlocal
+
+import (
+	"bufio"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+)
+
+func ParseClaudeJSONL(path, workspaceRoot string) ([]LocalToolUsageEvent, error) {
+	type candidate struct {
+		event    LocalToolUsageEvent
+		stopDone bool
+		score    int64
+	}
+
+	best := map[string]candidate{}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+
+		var row map[string]any
+		if err := json.Unmarshal([]byte(line), &row); err != nil {
+			continue
+		}
+		if strings.TrimSpace(asString(row["type"])) != "assistant" {
+			continue
+		}
+		if filepath.Clean(asString(row["cwd"])) != filepath.Clean(workspaceRoot) {
+			continue
+		}
+
+		msg, _ := row["message"].(map[string]any)
+		msgID := strings.TrimSpace(asString(msg["id"]))
+		sessionID := strings.TrimSpace(asString(row["sessionId"]))
+		if msgID == "" || sessionID == "" {
+			continue
+		}
+
+		usage, _ := msg["usage"].(map[string]any)
+		score := asInt64(usage["input_tokens"]) + asInt64(usage["output_tokens"]) +
+			asInt64(usage["cache_creation_input_tokens"]) + asInt64(usage["cache_read_input_tokens"])
+
+		cur := candidate{
+			event: LocalToolUsageEvent{
+				Tool:              "claude",
+				ToolSessionID:     sessionID,
+				ToolEventID:       msgID,
+				DedupeKey:         "claude:" + sessionID + ":" + msgID,
+				RequestCount:      1,
+				UsageUnit:         UsageUnitToken,
+				InputTokens:       asInt64(usage["input_tokens"]),
+				OutputTokens:      asInt64(usage["output_tokens"]),
+				CachedInputTokens: asInt64(usage["cache_creation_input_tokens"]) + asInt64(usage["cache_read_input_tokens"]),
+				RawSourcePath:     path,
+				RawPayload:        row,
+			},
+			stopDone: strings.TrimSpace(asString(row["stop_reason"])) == "end_turn",
+			score:    score,
+		}
+
+		old, ok := best[cur.event.DedupeKey]
+		if !ok || (cur.stopDone && !old.stopDone) || (cur.stopDone == old.stopDone && cur.score >= old.score) {
+			best[cur.event.DedupeKey] = cur
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	out := make([]LocalToolUsageEvent, 0, len(best))
+	for _, item := range best {
+		out = append(out, item.event)
+	}
+	slices.SortFunc(out, func(a, b LocalToolUsageEvent) int { return strings.Compare(a.DedupeKey, b.DedupeKey) })
+	return out, nil
+}

ae-cli/internal/attributionlocal/claude_jsonl_test.go

@@ -0,0 +1,23 @@
+package attributionlocal
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseClaudeJSONL_PrefersEndTurnRecord(t *testing.T) {
+	t.Parallel()
+
+	path := writeFile(t, "claude.jsonl", strings.Join([]string{
+		`{"type":"assistant","cwd":"/tmp/repo","sessionId":"claude-1","message":{"id":"msg-1","usage":{"input_tokens":5,"output_tokens":1}},"stop_reason":null}`,
+		`{"type":"assistant","cwd":"/tmp/repo","sessionId":"claude-1","message":{"id":"msg-1","usage":{"input_tokens":5,"output_tokens":3}},"stop_reason":"end_turn"}`,
+	}, "\n"))
+
+	events, err := ParseClaudeJSONL(path, "/tmp/repo")
+	if err != nil {
+		t.Fatalf("ParseClaudeJSONL: %v", err)
+	}
+	if len(events) != 1 || events[0].OutputTokens != 3 {
+		t.Fatalf("events = %+v", events)
+	}
+}

ae-cli/internal/attributionlocal/codex_jsonl.go

@@ -0,0 +1,78 @@
+package attributionlocal
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func ParseCodexJSONLFallback(path, workspaceRoot string) ([]LocalToolUsageEvent, error) {
+	lines, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var sessionID string
+	var events []LocalToolUsageEvent
+	for _, raw := range strings.Split(string(lines), "\n") {
+		raw = strings.TrimSpace(raw)
+		if raw == "" {
+			continue
+		}
+
+		var row map[string]any
+		if err := json.Unmarshal([]byte(raw), &row); err != nil {
+			continue
+		}
+
+		switch strings.TrimSpace(asString(row["type"])) {
+		case "session_meta":
+			payload, _ := row["payload"].(map[string]any)
+			if filepath.Clean(asString(payload["cwd"])) == filepath.Clean(workspaceRoot) {
+				sessionID = strings.TrimSpace(asString(payload["id"]))
+			}
+		case "event_msg":
+			if sessionID == "" {
+				continue
+			}
+			payload, _ := row["payload"].(map[string]any)
+			if strings.TrimSpace(asString(payload["type"])) != "token_count" {
+				continue
+			}
+			info, _ := payload["info"].(map[string]any)
+			lastUsage, _ := info["last_token_usage"].(map[string]any)
+			totalUsage, _ := info["total_token_usage"].(map[string]any)
+			selected := lastUsage
+			if len(selected) == 0 {
+				selected = totalUsage
+			}
+			if len(selected) == 0 {
+				continue
+			}
+			responseID := strings.TrimSpace(asString(payload["response_id"]))
+			if responseID == "" {
+				responseID = filepath.Base(path)
+			}
+			events = append(events, LocalToolUsageEvent{
+				Tool:              "codex",
+				ToolSessionID:     sessionID,
+				ToolEventID:       responseID,
+				DedupeKey:         "codex-jsonl:" + sessionID + ":" + responseID,
+				UsageUnit:         UsageUnitToken,
+				RequestCount:      1,
+				InputTokens:       asInt64(selected["input_tokens"]),
+				OutputTokens:      asInt64(selected["output_tokens"]),
+				CachedInputTokens: asInt64(selected["cached_input_tokens"]),
+				ReasoningTokens:   asInt64(selected["reasoning_output_tokens"]),
+				RawSourcePath:     path,
+				RawPayload:        payload,
+			})
+		}
+	}
+
+	if len(events) == 0 {
+		return nil, nil
+	}
+	return events, nil
+}

ae-cli/internal/attributionlocal/codex_jsonl_test.go

@@ -0,0 +1,44 @@
+package attributionlocal
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseCodexJSONL_PrefersLastTokenUsage(t *testing.T) {
+	t.Parallel()
+
+	path := writeFile(t, "codex.jsonl", strings.Join([]string{
+		`{"type":"session_meta","payload":{"id":"sess-1","cwd":"/tmp/repo"}}`,
+		`{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":100,"cached_input_tokens":10,"output_tokens":20,"reasoning_output_tokens":5,"total_tokens":120},"last_token_usage":{"input_tokens":7,"cached_input_tokens":1,"output_tokens":2,"reasoning_output_tokens":1,"total_tokens":9}}}}`,
+	}, "\n"))
+
+	events, err := ParseCodexJSONLFallback(path, "/tmp/repo")
+	if err != nil {
+		t.Fatalf("ParseCodexJSONLFallback: %v", err)
+	}
+	if len(events) != 1 || events[0].InputTokens != 7 || events[0].OutputTokens != 2 {
+		t.Fatalf("events = %+v", events)
+	}
+}
+
+func TestParseCodexJSONL_EmitsMultipleEventsPerFile(t *testing.T) {
+	t.Parallel()
+
+	path := writeFile(t, "codex.jsonl", strings.Join([]string{
+		`{"type":"session_meta","payload":{"id":"sess-1","cwd":"/tmp/repo"}}`,
+		`{"type":"event_msg","payload":{"type":"token_count","response_id":"resp-1","info":{"last_token_usage":{"input_tokens":7,"cached_input_tokens":1,"output_tokens":2,"reasoning_output_tokens":1,"total_tokens":9}}}}`,
+		`{"type":"event_msg","payload":{"type":"token_count","response_id":"resp-2","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1,"total_tokens":12}}}}`,
+	}, "\n"))
+
+	events, err := ParseCodexJSONLFallback(path, "/tmp/repo")
+	if err != nil {
+		t.Fatalf("ParseCodexJSONLFallback: %v", err)
+	}
+	if len(events) != 2 {
+		t.Fatalf("event count = %d, want 2", len(events))
+	}
+	if events[0].DedupeKey == events[1].DedupeKey {
+		t.Fatalf("dedupe keys should differ: %+v", events)
+	}
+}