Merge pull request #37 from x3c3/feat/rate-limit-llm-calls

feat: add rate limiting for LLM API calls

Author: dacharyc

evaluate/evaluate.go

@@ -47,6 +47,7 @@ type Options struct {
 	MaxLen    int
 	CacheDir  string       // Override cache directory; defaults to judge.CacheDir(skillDir) when empty
 	Progress  ProgressFunc // Optional progress callback; nil means no output
+	RateLimit int          // Max LLM API requests per second; 0 means unlimited
 }
 
 // progress calls the progress callback if set.
@@ -65,6 +66,17 @@ func resolveCacheDir(opts Options, skillDir string) string {
 	return judge.CacheDir(skillDir)
 }
 
+// newThrottle returns a function that blocks until the next request is allowed.
+// If rps is 0 or negative, the returned function is a no-op.
+// The caller must call the returned stop function when done.
+func newThrottle(rps int) (wait func(), stop func()) {
+	if rps <= 0 {
+		return func() {}, func() {}
+	}
+	ticker := time.NewTicker(time.Second / time.Duration(rps))
+	return func() { <-ticker.C }, ticker.Stop
+}
+
 // EvaluateSkill scores a skill directory (SKILL.md and/or reference files).
 func EvaluateSkill(ctx context.Context, dir string, client judge.LLMClient, opts Options) (*Result, error) {
 	result := &Result{SkillDir: dir}
@@ -77,6 +89,9 @@ func EvaluateSkill(ctx context.Context, dir string, client judge.LLMClient, opts
 		return nil, fmt.Errorf("loading skill: %w", err)
 	}
 
+	wait, stop := newThrottle(opts.RateLimit)
+	defer stop()
+
 	// Score SKILL.md
 	if !opts.RefsOnly {
 		progress(opts, "scoring", fmt.Sprintf("%s/SKILL.md", skillName))
@@ -94,6 +109,7 @@ func EvaluateSkill(ctx context.Context, dir string, client judge.LLMClient, opts
 		}
 
 		if result.SkillScores == nil {
+			wait()
 			scores, err := judge.ScoreSkill(ctx, s.RawContent, client, opts.MaxLen)
 			if err != nil {
 				return nil, fmt.Errorf("scoring SKILL.md: %w", err)
@@ -148,6 +164,7 @@ func EvaluateSkill(ctx context.Context, dir string, client judge.LLMClient, opts
 				}
 
 				if refScores == nil {
+					wait()
 					scores, err := judge.ScoreReference(ctx, content, s.Frontmatter.Name, skillDesc, client, opts.MaxLen)
 					if err != nil {
 						progress(opts, "error", fmt.Sprintf("scoring %s: %v", name, err))
@@ -235,6 +252,10 @@ func EvaluateSingleFile(ctx context.Context, absPath string, client judge.LLMCli
 		}
 	}
 
+	wait, stop := newThrottle(opts.RateLimit)
+	defer stop()
+
+	wait()
 	scores, err := judge.ScoreReference(ctx, string(content), skillName, s.Frontmatter.Description, client, opts.MaxLen)
 	if err != nil {
 		return nil, fmt.Errorf("scoring %s: %w", fileName, err)

evaluate/evaluate_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/agent-ecosystem/skill-validator/judge"
 )
@@ -492,3 +493,43 @@ func TestEvaluateSingleFile_LLMError(t *testing.T) {
 		t.Errorf("unexpected error: %v", err)
 	}
 }
+
+func TestEvaluateSkill_RateLimiting(t *testing.T) {
+	dir := makeSkillDir(t, map[string]string{
+		"a.md": "# A",
+		"b.md": "# B",
+		"c.md": "# C",
+	})
+	client := &mockLLMClient{responses: []string{skillJSON, refJSON, refJSON, refJSON}}
+
+	// 5 req/s = 200ms between calls. 4 calls (1 skill + 3 refs) = at least 600ms.
+	start := time.Now()
+	_, err := EvaluateSkill(context.Background(), dir, client, Options{
+		MaxLen:    8000,
+		RateLimit: 5,
+	})
+	elapsed := time.Since(start)
+	if err != nil {
+		t.Fatalf("EvaluateSkill error = %v", err)
+	}
+	// 4 API calls at 5/s means 3 intervals of 200ms = 600ms minimum
+	if elapsed < 500*time.Millisecond {
+		t.Errorf("expected >= 500ms for rate-limited calls, got %v", elapsed)
+	}
+}
+
+func TestEvaluateSkill_RateLimitZeroDisabled(t *testing.T) {
+	dir := makeSkillDir(t, map[string]string{"a.md": "# A"})
+	client := &mockLLMClient{responses: []string{skillJSON, refJSON}}
+
+	start := time.Now()
+	_, err := EvaluateSkill(context.Background(), dir, client, Options{MaxLen: 8000})
+	elapsed := time.Since(start)
+	if err != nil {
+		t.Fatalf("EvaluateSkill error = %v", err)
+	}
+	// With no rate limit (default 0), should complete quickly
+	if elapsed > 2*time.Second {
+		t.Errorf("expected fast completion without rate limit, got %v", elapsed)
+	}
+}