Skip to content

Commit 60e6348

Browse files
fix(generate): configurable per-batch timeout + batch size + .spectra-debug.log
The 5-minute SDK timeout in CopilotGenerationAgent has been hardcoded since spec 009. Fast generators (gpt-4o, gpt-4o-mini) fit 30 tests per batch in under 5 minutes; slower / reasoning models or large Azure deployments do not. Symptom: --count 100 splits into 4 batches of 30, the first batch hits the 5-minute ceiling, generation fails. --count 1 works because one test fits in any budget. AiConfig (new optional fields): - generation_timeout_minutes (default 5) — per-batch SDK call timeout - generation_batch_size (default 30) — number of tests per AI call - debug_log_enabled (default true) — gates the per-batch diagnostic file GenerationAgent.GenerateTestsAsync: - Reads timeout from config; minimum 1 minute - Logs each batch START / OK / TIMEOUT to .spectra-debug.log with the configured timeout, model name, provider, requested count, and elapsed seconds (best-effort, never throws) - Surfaces the configured timeout and batch size in the live status message - Timeout error message now lists the actual configured minutes, the model name, the batch size, and three concrete remediation options with copy-paste-ready spectra.config.json snippets GenerateHandler: - Replaces hardcoded BatchSize const with DefaultBatchSize fallback - Reads ai.generation_batch_size from config when > 0 - Renders "Batch N/M" in progress messages (was "Batch N") - Loop math unchanged For users hitting the timeout with a slow model, the workaround is now in config — no code change required: "ai": { "generation_timeout_minutes": 15, "generation_batch_size": 10 } The .spectra-debug.log file shows exactly which batch was slow and why.
1 parent 0da74c6 commit 60e6348

File tree

3 files changed

+80
-8
lines changed

3 files changed

+80
-8
lines changed

src/Spectra.CLI/Agent/Copilot/GenerationAgent.cs

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,15 +170,23 @@ public async Task<GenerationResult> GenerateTestsAsync(
170170
var promptLoader = new PromptTemplateLoader(_basePath);
171171
var fullPrompt = BuildFullPrompt(prompt, requestedCount, criteriaContext, templateLoader: promptLoader, profileFormat: profileFormat, testimizeEnabled: _config.Testimize.Enabled && testimizeClient is not null);
172172

173-
// Send and wait for the complete response
174-
_onStatus?.Invoke("Starting AI generation...");
173+
// Send and wait for the complete response.
174+
// The per-batch timeout is configurable via ai.generation_timeout_minutes.
175+
// Slower / reasoning models may need 10–20+ minutes per batch.
176+
var timeoutMinutes = Math.Max(1, _config.Ai.GenerationTimeoutMinutes);
177+
var batchTimeout = TimeSpan.FromMinutes(timeoutMinutes);
178+
var sw = System.Diagnostics.Stopwatch.StartNew();
179+
DebugLog($"BATCH START requested={requestedCount} model={_provider?.Model ?? "?"} provider={_provider?.Name ?? "?"} timeout={timeoutMinutes}min");
180+
_onStatus?.Invoke($"Starting AI generation ({requestedCount} tests, timeout {timeoutMinutes} min)...");
175181
var response = await session.SendAndWaitAsync(
176182
new MessageOptions { Prompt = fullPrompt },
177-
timeout: TimeSpan.FromMinutes(5),
183+
timeout: batchTimeout,
178184
cancellationToken: ct);
179185

180186
// Cancel delayed timers so they don't overwrite the final result
181187
await timerCts.CancelAsync();
188+
sw.Stop();
189+
DebugLog($"BATCH OK requested={requestedCount} elapsed={sw.Elapsed.TotalSeconds:F1}s");
182190

183191
var responseText = response?.Data?.Content ?? "";
184192

@@ -219,10 +227,21 @@ public async Task<GenerationResult> GenerateTestsAsync(
219227
}
220228
catch (TimeoutException)
221229
{
230+
var configuredMinutes = Math.Max(1, _config.Ai.GenerationTimeoutMinutes);
231+
DebugLog($"BATCH TIMEOUT requested={requestedCount} model={_provider?.Model ?? "?"} configured_timeout={configuredMinutes}min");
222232
return new GenerationResult
223233
{
224234
Tests = [],
225-
Errors = ["Generation timed out after 5 minutes. Try reducing --count."]
235+
Errors = [
236+
$"Generation timed out after {configuredMinutes} minutes (model: {_provider?.Model ?? "?"}, batch size: {requestedCount}).",
237+
"Options to fix this:",
238+
" 1. Increase the per-batch timeout in spectra.config.json:",
239+
" \"ai\": { \"generation_timeout_minutes\": 15 }",
240+
" 2. Reduce the batch size:",
241+
" \"ai\": { \"generation_batch_size\": 10 }",
242+
" 3. Reduce --count or use a faster model.",
243+
"See .spectra-debug.log for per-batch timing details."
244+
]
226245
};
227246
}
228247
catch (Exception ex) when (ex.Message.Contains("copilot", StringComparison.OrdinalIgnoreCase)
@@ -269,6 +288,26 @@ public async Task<GenerationResult> GenerateTestsAsync(
269288
}
270289
}
271290

291+
/// <summary>
292+
/// Append a one-line timestamped diagnostic to <c>.spectra-debug.log</c>
293+
/// in the project root. Best-effort; never throws. Gated by
294+
/// <c>ai.debug_log_enabled</c> (default true).
295+
/// </summary>
296+
private void DebugLog(string message)
297+
{
298+
try
299+
{
300+
if (!_config.Ai.DebugLogEnabled) return;
301+
var path = Path.Combine(_basePath, ".spectra-debug.log");
302+
var line = $"{DateTime.UtcNow:yyyy-MM-ddTHH:mm:ss.fffZ} [generate] {message}{Environment.NewLine}";
303+
File.AppendAllText(path, line);
304+
}
305+
catch
306+
{
307+
// Diagnostics must never block generation.
308+
}
309+
}
310+
272311
private void SaveDebugResponse(string responseText)
273312
{
274313
try

src/Spectra.CLI/Commands/Generate/GenerateHandler.cs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ namespace Spectra.CLI.Commands.Generate;
3232
/// </summary>
3333
public sealed class GenerateHandler
3434
{
35-
private const int BatchSize = 30;
35+
private const int DefaultBatchSize = 30;
3636

3737
private readonly VerbosityLevel _verbosity;
3838
private readonly bool _dryRun;
@@ -522,14 +522,22 @@ private async Task<int> ExecuteDirectModeAsync(
522522
var batchErrors = new List<string>();
523523
var batchesCompleted = 0;
524524

525+
// Spec post-038 fix: batch size is configurable via ai.generation_batch_size.
526+
// Slower / reasoning models benefit from smaller batches paired with a
527+
// larger ai.generation_timeout_minutes setting.
528+
var configuredBatchSize = config.Ai.GenerationBatchSize > 0
529+
? config.Ai.GenerationBatchSize
530+
: DefaultBatchSize;
531+
var totalBatches = (int)Math.Ceiling(effectiveCount / (double)configuredBatchSize);
532+
525533
for (var batchNum = 1; totalGenerated < effectiveCount; batchNum++)
526534
{
527535
var remaining = effectiveCount - totalGenerated;
528-
var batchRequestCount = Math.Min(remaining, BatchSize);
536+
var batchRequestCount = Math.Min(remaining, configuredBatchSize);
529537

530538
UpdateProgress(suite, "generating",
531-
$"Generating batch {batchNum}: {allWrittenTests.Count}/{effectiveCount} tests complete...");
532-
_progress.Info($"Batch {batchNum}: requesting {batchRequestCount} tests ({allWrittenTests.Count}/{effectiveCount} complete)");
539+
$"Generating batch {batchNum}/{totalBatches}: {allWrittenTests.Count}/{effectiveCount} tests complete...");
540+
_progress.Info($"Batch {batchNum}/{totalBatches}: requesting {batchRequestCount} tests ({allWrittenTests.Count}/{effectiveCount} complete)");
533541

534542
var prompt = BuildPrompt(suite, batchRequestCount, mutableExistingIds, effectiveProfile, focus);
535543
GenerationResult batchResult = null!;

src/Spectra.Core/Models/Config/AiConfig.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,29 @@ public sealed class AiConfig
1919
/// </summary>
2020
[JsonPropertyName("critic")]
2121
public CriticConfig? Critic { get; init; }
22+
23+
/// <summary>
24+
/// Per-batch timeout for the AI generation SDK call, in minutes. Default 5.
25+
/// Slower / larger models (e.g. reasoning models, large Azure deployments)
26+
/// may need to bump this to 10–20+ minutes. The timer measures the entire
27+
/// batch round-trip including all tool calls the AI makes.
28+
/// </summary>
29+
[JsonPropertyName("generation_timeout_minutes")]
30+
public int GenerationTimeoutMinutes { get; init; } = 5;
31+
32+
/// <summary>
33+
/// Number of tests requested per AI call. Default 30. Smaller batches
34+
/// reduce per-batch latency on slow models at the cost of more total
35+
/// round-trips. Pair with <see cref="GenerationTimeoutMinutes"/>.
36+
/// </summary>
37+
[JsonPropertyName("generation_batch_size")]
38+
public int GenerationBatchSize { get; init; } = 30;
39+
40+
/// <summary>
41+
/// Append per-batch diagnostics to <c>.spectra-debug.log</c> in the
42+
/// project root. Default true. Useful for diagnosing slow models and
43+
/// timeout issues. Set false to silence.
44+
/// </summary>
45+
[JsonPropertyName("debug_log_enabled")]
46+
public bool DebugLogEnabled { get; init; } = true;
2247
}

0 commit comments

Comments
 (0)