diff --git a/docs/internal/capacity.md b/docs/internal/capacity.md index 9101bad..09af4a5 100644 --- a/docs/internal/capacity.md +++ b/docs/internal/capacity.md @@ -56,7 +56,10 @@ reserve another bounded slot. `NewClient` wraps the engine's selected internal model client after public client adaptation or built-in client construction. Initial generation and the -default repairer receive the same wrapper. +default repairer receive the same wrapper. Their requests retain the same +effective backend, credential, numeric-presence metadata, and structured-output +settings, so scheduling does not change provider omission semantics between +calls. For each `Generate` call, the wrapper selects a pool from the request's effective backend ID. An unlimited call passes directly to the next client. A @@ -71,7 +74,9 @@ other backend IDs. The wrapper passes generation requests, responses, and collaborator errors through unchanged. It owns scheduling only; the concrete model client remains -responsible for provider transport behavior. +responsible for provider transport behavior. The runner, rather than the +capacity layer, sums all five usage fields from the initial response and every +completed repair response into the successful run result. ## Cancellation And Release diff --git a/docs/internal/runner.md b/docs/internal/runner.md index e170167..6977c09 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -144,20 +144,27 @@ each actual generation call. When an internal repairer is present, a JSON or JSON Schema content failure can trigger bounded repair attempts. Repair receives the effective execution -target and session ID, validation errors, prior output, and structured-output -specification. The default repairer uses the same wrapped client as initial -generation, so each repair reacquires the selected backend's active permit -while remaining inside its original admission lease. Repair never performs a -second bounded admission, and repaired outputs use the operation's existing -validation plan. This capability remains internal and is not a public option. +target, explicit numeric-presence bits, credential, backend identity, session +ID, validation errors, prior output, and structured-output specification. One +request constructor supplies those common fields to initial and repair +generation while their rendered prompts remain intentionally distinct. The +default repairer uses the same wrapped client as initial generation, so each +repair reacquires the selected backend's active permit while remaining inside +its original admission lease. Repair never performs a second bounded +admission, and repaired outputs use the operation's existing validation plan. +This capability remains internal and is not a public option. A successful result includes the output artifact and raw output, validation state, effective session ID, prompt and rendered-prompt hashes, selected profile and backend, effective settings, input hashes, token usage, a generated run identifier, and UTC timing. The same effective session reaches initial generation and any repair attempt through the rendered prompt. The same -effective target, including backend identity, reaches generation and any -repair attempt. +effective target and presence metadata, including backend identity and direct +credential during execution, reaches generation and every repair attempt. +Result usage is the field-wise sum of all five usage values from the initial +response and every completed repair response. Final raw output, artifact, and +validation state still come from the last candidate. A repair error returns no +partial run result or partial usage. ## Failure Categories @@ -193,8 +200,9 @@ The [runner tests](../../internal/usecase/runner_test.go) own preparation order, selection and override precedence, the two-phase boundary, early admission, lease lifetime and release, direct-session resolution, schema-before-generation behavior, hashing, generation and validation outcomes, backend propagation, -bounded repair, shared initial/repair capacity, credentials and redaction, -error categories, artifact metadata, usage, and timing. The +bounded repair progression, initial/repair request parity, cumulative usage, +shared initial/repair capacity, credentials and redaction, error categories, +artifact metadata, and timing. The [capacity subsystem document](capacity.md) identifies the focused pool, waiter, and wrapped-client tests. diff --git a/internal/usecase/generation_request.go b/internal/usecase/generation_request.go new file mode 100644 index 0000000..cccf36b --- /dev/null +++ b/internal/usecase/generation_request.go @@ -0,0 +1,19 @@ +package usecase + +import "gitea.maximumdirect.net/eric/promptkit/internal/domain" + +func newGenerationRequest( + prompt domain.RenderedPrompt, + sessionID string, + target domain.ExecutionTarget, + targetPresence domain.ExecutionTargetPresence, + structuredOutput *domain.StructuredOutputSpec, +) domain.GenerateRequest { + prompt.SessionID = sessionID + return domain.GenerateRequest{ + Prompt: prompt, + Target: target, + TargetPresence: targetPresence, + StructuredOutput: structuredOutput, + } +} diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index 427cbee..e00d2f5 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -317,7 +317,13 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test }, } repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}}, + responses: []*domain.GenerateResponse{{ + Content: `{"repaired":true}`, + Usage: domain.TokenUsage{ + PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, + CachedTokens: 7, CacheWriteTokens: 11, + }, + }}, } admitter := &fakeRunAdmitter{} reader := defaultArtifactReader() @@ -328,7 +334,13 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test nil, reader, renderer, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}}, + &fakeLLM{resp: &domain.GenerateResponse{ + Content: `{"broken":true}`, + Usage: domain.TokenUsage{ + PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, + CachedTokens: 23, CacheWriteTokens: 29, + }, + }}, validator, repairer, admitter, @@ -355,6 +367,13 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 { t.Fatalf("unexpected repaired validation result: %+v", result.Validation) } + wantUsage := domain.TokenUsage{ + PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, + CachedTokens: 30, CacheWriteTokens: 40, + } + if result.Usage != wantUsage { + t.Fatalf("prepared cumulative usage = %+v, want %+v", result.Usage, wantUsage) + } if admitter.releaseCalls != 1 { t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls) } diff --git a/internal/usecase/repairer.go b/internal/usecase/repairer.go index a705c74..587576f 100644 --- a/internal/usecase/repairer.go +++ b/internal/usecase/repairer.go @@ -19,6 +19,7 @@ type RepairRequest struct { ValidationErrors []string SessionID string Target domain.ExecutionTarget + TargetPresence domain.ExecutionTargetPresence StructuredOutput *domain.StructuredOutputSpec Attempt int MaxAttempts int @@ -44,7 +45,6 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) ( } prompt := domain.RenderedPrompt{ - SessionID: req.SessionID, Messages: []domain.RenderedMessage{ { Role: "system", @@ -64,11 +64,13 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) ( }, } - resp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: prompt, - Target: req.Target, - StructuredOutput: req.StructuredOutput, - }) + resp, err := r.llm.Generate(ctx, newGenerationRequest( + prompt, + req.SessionID, + req.Target, + req.TargetPresence, + req.StructuredOutput, + )) if err != nil { return nil, err } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 6243839..c63d127 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -175,18 +175,20 @@ func (r *Runner) executePreparedRun( ) (*domain.RunResult, error) { executionTarget := prepared.EffectiveModelParams executionTarget.APIKey = directAPIKey - genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages}, - Target: executionTarget, - TargetPresence: prepared.TargetPresence, - StructuredOutput: prepared.StructuredOutput, - }) + genResp, err := r.llm.Generate(ctx, newGenerationRequest( + domain.RenderedPrompt{Messages: prepared.Messages}, + prepared.SessionID, + executionTarget, + prepared.TargetPresence, + prepared.StructuredOutput, + )) if err != nil { if errors.Is(err, llm.ErrInvalidRequest) { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) } return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) } + usage := genResp.Usage outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) validationResult, err := validateArtifact(ctx, &outputArtifact, 0) @@ -204,6 +206,7 @@ func (r *Runner) executePreparedRun( ValidationErrors: validationResult.Errors, SessionID: prepared.SessionID, Target: executionTarget, + TargetPresence: prepared.TargetPresence, StructuredOutput: prepared.StructuredOutput, Attempt: attemptsUsed, MaxAttempts: prepared.OutputContract.RepairAttempts, @@ -217,6 +220,7 @@ func (r *Runner) executePreparedRun( } genResp = repairResp + usage = addTokenUsage(usage, repairResp.Usage) outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed) @@ -245,13 +249,23 @@ func (r *Runner) executePreparedRun( Endpoint: prepared.EffectiveModelParams.Endpoint, EffectiveModelParams: executionTarget, InputHashes: prepared.InputHashes, - Usage: genResp.Usage, + Usage: usage, StartTime: start, EndTime: end, Duration: end.Sub(start), }, nil } +func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage { + return domain.TokenUsage{ + PromptTokens: total.PromptTokens + next.PromptTokens, + CompletionTokens: total.CompletionTokens + next.CompletionTokens, + TotalTokens: total.TotalTokens + next.TotalTokens, + CachedTokens: total.CachedTokens + next.CachedTokens, + CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens, + } +} + func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) { state, err := r.resolvePreparation(ctx, req, time.Now().UTC()) if err != nil { diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index edcb083..bd2e3e5 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -228,6 +228,30 @@ type fakeRepairer struct { reqs []RepairRequest } +type sequenceLLM struct { + responses []*domain.GenerateResponse + requests []domain.GenerateRequest +} + +func (c *sequenceLLM) Generate(_ context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + c.requests = append(c.requests, req) + index := len(c.requests) - 1 + if index >= len(c.responses) { + return nil, errors.New("no generation response configured") + } + return c.responses[index], nil +} + +type recordingRepairer struct { + next OutputRepairer + reqs []RepairRequest +} + +func (r *recordingRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { + r.reqs = append(r.reqs, req) + return r.next.Repair(ctx, req) +} + type fakeRunAdmitter struct { backendIDs []string err error @@ -2070,50 +2094,255 @@ func TestRunnerRunValidationStillWorks(t *testing.T) { } } -func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t *testing.T) { - repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}} +func TestRunnerRepairStateMachine(t *testing.T) { + failed := func(mode domain.ValidationMode, diagnostic string) domain.ValidationResult { + return domain.ValidationResult{ + Status: domain.ValidationFailed, + Mode: mode, + Errors: []string{diagnostic}, + IsValid: false, + } + } + passed := func(mode domain.ValidationMode) domain.ValidationResult { + return domain.ValidationResult{ + Status: domain.ValidationPassed, + Mode: mode, + IsValid: true, + } + } + responses := func(count int) []*domain.GenerateResponse { + values := make([]*domain.GenerateResponse, count) + for index := range values { + unit := index + 1 + values[index] = &domain.GenerateResponse{ + Content: fmt.Sprintf(`{"candidate":%d}`, index), + Usage: domain.TokenUsage{ + PromptTokens: unit, + CompletionTokens: unit * 10, + TotalTokens: unit * 100, + CachedTokens: unit * 1000, + CacheWriteTokens: unit * 10000, + }, + } + } + return values + } + zeroOverrides := &domain.ExecutionTargetOverride{ + Temperature: float64Ptr(0), + MaxTokens: intPtr(0), + TopP: float64Ptr(0), + TimeoutSeconds: intPtr(0), + } + allPresent := domain.ExecutionTargetPresence{ + Temperature: true, + MaxTokens: true, + TopP: true, + TimeoutSeconds: true, + } - runner := NewRunnerWithRepairer( - &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55}, - }}, fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend/v1"}, - }}, + tests := []struct { + name string + mode domain.ValidationMode + budget int + validationResults []domain.ValidationResult + responses []*domain.GenerateResponse + execution *domain.ExecutionTargetOverride + wantPresence domain.ExecutionTargetPresence + wantRepairs int + wantStatus domain.ValidationStatus + structured bool + }{ + { + name: "initial success does not repair", + mode: domain.ValidationJSON, + budget: 3, + validationResults: []domain.ValidationResult{passed(domain.ValidationJSON)}, + responses: responses(1), + wantStatus: domain.ValidationPassed, + }, + { + name: "basic failure is ineligible despite budget", + mode: domain.ValidationBasic, + budget: 3, + validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output")}, + responses: responses(1), + wantStatus: domain.ValidationFailed, + }, + { + name: "inherited numeric values remain absent", + mode: domain.ValidationJSON, + budget: 1, + validationResults: []domain.ValidationResult{ + failed(domain.ValidationJSON, "initial syntax"), + passed(domain.ValidationJSON), + }, + responses: responses(2), + wantRepairs: 1, + wantStatus: domain.ValidationPassed, + }, + { + name: "explicit numeric zeros remain present", + mode: domain.ValidationJSONSchema, + budget: 1, + execution: zeroOverrides, + structured: true, + validationResults: []domain.ValidationResult{ + failed(domain.ValidationJSONSchema, "initial schema mismatch"), + passed(domain.ValidationJSONSchema), + }, + responses: responses(2), + wantPresence: allPresent, + wantRepairs: 1, + wantStatus: domain.ValidationPassed, + }, + { + name: "successful repair stops below larger budget", + mode: domain.ValidationJSON, + budget: 4, + validationResults: []domain.ValidationResult{ + failed(domain.ValidationJSON, "candidate zero"), + failed(domain.ValidationJSON, "candidate one"), + passed(domain.ValidationJSON), + }, + responses: responses(3), + wantRepairs: 2, + wantStatus: domain.ValidationPassed, + }, + { + name: "failed repairs exhaust exact larger budget", + mode: domain.ValidationJSON, + budget: 3, + validationResults: []domain.ValidationResult{ + failed(domain.ValidationJSON, "candidate zero"), + failed(domain.ValidationJSON, "candidate one"), + failed(domain.ValidationJSON, "candidate two"), + failed(domain.ValidationJSON, "candidate three"), + }, + responses: responses(4), + wantRepairs: 3, + wantStatus: domain.ValidationFailed, + }, + } - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validate.NewStandardValidator("."), - repairer, nil) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + definition := promptDef(domain.FormatJSON, tc.mode, tc.budget) + plan := &recordingPreparedValidation{results: tc.validationResults} + if tc.structured { + definition.Validation.SchemaPath = "schema.json" + plan.schemaDocument = map[string]any{"type": "object"} + } + validator := &recordingValidationPreparer{plan: plan} + client := &sequenceLLM{responses: tc.responses} + repairer := &recordingRepairer{next: NewDefaultOutputRepairer(client)} + runner := NewRunnerWithRepairer( + &fakePromptRepo{def: definition}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", BackendID: "custom", Model: "profile-model"}, + }}, + fakeBackendResolver{backends: map[string]domain.Backend{ + "custom": {ID: "custom", Endpoint: "http://backend.example/v1"}, + }}, + defaultArtifactReader(), + defaultRenderer(), + client, + validator, + repairer, + nil, + ) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if repairer.calls != 1 || res.Validation.RepairAttempts != 1 { - t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts) - } - if len(repairer.reqs) != 1 { - t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) - } - if repairer.reqs[0].Target.Endpoint != "http://override/v1" || repairer.reqs[0].Target.Model != "override-model" { - t.Fatalf("expected repair to use effective target, got %+v", repairer.reqs[0].Target) - } - if repairer.reqs[0].Target.TimeoutSeconds != 22 { - t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds) - } - if llmClient.lastReq.Target.BackendID != "custom" || - repairer.reqs[0].Target.BackendID != "custom" || - res.SelectedBackendID != "custom" { - t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q", - llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID) + result, err := runner.Run(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + SessionID: " repair-session ", + APIKey: "direct-secret", + Inputs: singleInputRef(), + Execution: tc.execution, + }) + if err != nil { + t.Fatalf("run: %v", err) + } + if len(client.requests) != tc.wantRepairs+1 || len(repairer.reqs) != tc.wantRepairs { + t.Fatalf( + "generation/repair calls = (%d, %d), want (%d, %d)", + len(client.requests), len(repairer.reqs), tc.wantRepairs+1, tc.wantRepairs, + ) + } + if len(plan.artifacts) != tc.wantRepairs+1 { + t.Fatalf("validation calls = %d, want %d", len(plan.artifacts), tc.wantRepairs+1) + } + + initialRequest := client.requests[0] + if initialRequest.TargetPresence != tc.wantPresence { + t.Fatalf("initial target presence = %+v, want %+v", initialRequest.TargetPresence, tc.wantPresence) + } + if initialRequest.Target.APIKey != "direct-secret" || initialRequest.Target.BackendID != "custom" || + initialRequest.Prompt.SessionID != "repair-session" { + t.Fatalf("initial common request fields = %+v", initialRequest) + } + if initialRequest.Target.Endpoint != "http://backend.example/v1" || + initialRequest.Target.Model != "profile-model" || + initialRequest.Target.Temperature != 0 || initialRequest.Target.MaxTokens != 0 || initialRequest.Target.TopP != 0 { + t.Fatalf("initial effective target = %+v", initialRequest.Target) + } + if tc.structured != (initialRequest.StructuredOutput != nil) { + t.Fatalf("initial structured output = %+v, want present %v", initialRequest.StructuredOutput, tc.structured) + } + if tc.structured && (initialRequest.StructuredOutput.JSONSchema == nil || + initialRequest.StructuredOutput.JSONSchema.Name != "p_1") { + t.Fatalf("initial JSON Schema metadata = %+v, want derived schema name p_1", initialRequest.StructuredOutput) + } + + for index, req := range repairer.reqs { + if req.Attempt != index+1 || req.MaxAttempts != tc.budget || req.Mode != tc.mode { + t.Fatalf("repair request %d progression = %+v", index, req) + } + if req.PreviousOutput != tc.responses[index].Content || + !reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) { + t.Fatalf("repair request %d prior state = %+v", index, req) + } + if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) || + req.SessionID != initialRequest.Prompt.SessionID || + !reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) { + t.Fatalf("repair request %d common fields drifted: %+v", index, req) + } + + generated := client.requests[index+1] + if generated.TargetPresence != initialRequest.TargetPresence || + !reflect.DeepEqual(generated.Target, initialRequest.Target) || + generated.Prompt.SessionID != initialRequest.Prompt.SessionID || + !reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) { + t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated) + } + if reflect.DeepEqual(generated.Prompt.Messages, initialRequest.Prompt.Messages) { + t.Fatalf("repair generation request %d reused the initial prompt", index) + } + } + + lastResponse := tc.responses[tc.wantRepairs] + if result.RawOutput != lastResponse.Content || string(result.Artifact.Body) != lastResponse.Content { + t.Fatalf("final output = (%q, %q), want %q", result.RawOutput, result.Artifact.Body, lastResponse.Content) + } + if result.Validation.Status != tc.wantStatus || result.Validation.RepairAttempts != tc.wantRepairs { + t.Fatalf("final validation = %+v, want status %q and %d repairs", result.Validation, tc.wantStatus, tc.wantRepairs) + } + if result.SelectedBackendID != "custom" || result.SessionID != "repair-session" || + result.EffectiveModelParams.APIKey != "" { + t.Fatalf("result execution metadata = %+v", result) + } + + var wantUsage domain.TokenUsage + for _, response := range tc.responses[:tc.wantRepairs+1] { + wantUsage.PromptTokens += response.Usage.PromptTokens + wantUsage.CompletionTokens += response.Usage.CompletionTokens + wantUsage.TotalTokens += response.Usage.TotalTokens + wantUsage.CachedTokens += response.Usage.CachedTokens + wantUsage.CacheWriteTokens += response.Usage.CacheWriteTokens + } + if result.Usage != wantUsage { + t.Fatalf("cumulative usage = %+v, want %+v", result.Usage, wantUsage) + } + }) } } @@ -2208,99 +2437,6 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi } } -func TestRunnerRunRepairCarriesEffectiveSessionID(t *testing.T) { - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}} - runner := NewRunnerWithRepairer( - &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://example.test/v1", Model: "model"}, - }}, - nil, - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validate.NewStandardValidator("."), - NewDefaultOutputRepairer(llmClient), nil) - - result, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - SessionID: " repair-session ", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if llmClient.calls != 2 { - t.Fatalf("expected initial generation and one repair, got %d calls", llmClient.calls) - } - if llmClient.lastReq.Prompt.SessionID != "repair-session" { - t.Fatalf("expected repair generation to retain effective session, got %q", llmClient.lastReq.Prompt.SessionID) - } - if result.SessionID != "repair-session" { - t.Fatalf("expected result to retain effective session, got %q", result.SessionID) - } -} - -func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { - def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1) - def.Validation.SchemaPath = "events.schema.json" - - validator := &fakeValidator{ - result: domain.ValidationResult{ - Status: domain.ValidationFailed, - Mode: domain.ValidationJSONSchema, - Errors: []string{"schema mismatch"}, - IsValid: false, - }, - schemaDoc: map[string]any{ - "type": "object", - "properties": map[string]any{ - "events": map[string]any{"type": "array"}, - }, - }, - } - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{ - {Content: `{"events":[]}`}, - }, - } - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}} - runner := NewRunnerWithRepairer( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, - - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validator, - repairer, nil) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput) - } - if len(repairer.reqs) != 1 { - t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) - } - if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil { - t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput) - } - if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" { - t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name) - } - if validator.schemaLoads != 1 || validator.validateCalls != 2 { - t.Fatalf("schema preparation/validation calls = (%d, %d), want (1, 2)", validator.schemaLoads, validator.validateCalls) - } -} - func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) { src := &domain.ExecutionProfile{ ID: "exec",