package llm import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" pk "gitea.maximumdirect.net/eric/promptkit" ) type reply struct { status int content string } func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) { return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{ "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", }) } func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) { documents := map[string]string{ "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", } if operatorPrompt != "" { documents["operator-prompt.yaml"] = operatorPrompt } return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents) } func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) { t.Helper() var count atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { call := int(count.Add(1)) if hook != nil { hook(r) } step := replies[len(replies)-1] if call <= len(replies) { step = replies[call-1] } if step.status == 0 { connection, _, hijackErr := w.(http.Hijacker).Hijack() if hijackErr != nil { t.Error(hijackErr) return } _ = connection.Close() return } if step.status != http.StatusOK { w.WriteHeader(step.status) _, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`) return } if step.content == "MALFORMED_ENVELOPE" { _, _ = fmt.Fprint(w, `{"choices":[]}`) return } encoded, _ := json.Marshal(step.content) w.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) })) t.Cleanup(server.Close) t.Setenv("ADAPTER_KEY", "adapter-secret") dir := t.TempDir() profileDirectory := filepath.Join(dir, "profiles") if err := os.Mkdir(profileDirectory, 0700); err != nil { t.Fatal(err) } base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs) main := filepath.Join(dir, "config.yml") if err := os.WriteFile(main, []byte(base), 0600); err != nil { t.Fatal(err) } for name, content := range documents { if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil { t.Fatal(err) } } loaded, err := config.Load(ck.File(main), "") if err != nil { t.Fatal(err) } runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client()) if err != nil { t.Fatal(err) } return runtime, &count, dir } func operatorPromptDocument(system, user string) string { return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user) } func adapterRequest(schema responseschema.Key) structuredoutput.Request { id := prompts.PromptIDModuleGlossaryProposal vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"} if schema == responseschema.ValidatorDecisionSetKey { id = prompts.PromptIDValidatorEditorialReview vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"} } return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} } func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { for _, schema := range []responseschema.Key{responseschema.CorrectionSetKey, responseschema.ValidatorDecisionSetKey} { valid := `{"corrections":[]}` if schema == responseschema.ValidatorDecisionSetKey { valid = `{"validations":[]}` } for _, tc := range []struct { name string replies []reply retries, repairs, attempts, repairCalls, transientRetries int malformed bool reason outcomes.ReasonCode cause error }{ {"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, "", nil}, {"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, "", nil}, {"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, outcomes.ReasonInvalidOutputExhausted, nil}, {"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil}, {"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil}, {"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderFailed, pk.ErrLLMGenerate}, {"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, outcomes.ReasonProviderFailed, pk.ErrRetriesExhausted}, {"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonTransportFailed, pk.ErrTransport}, {"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderResponseInvalid, pk.ErrMalformedResponse}, {"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, "", nil}, } { t.Run(string(schema)+"/"+tc.name, func(t *testing.T) { runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil) role := "proposal" if schema == responseschema.ValidatorDecisionSetKey { role = "validation" } var events []pk.ExecutionEvent var records []structuredoutput.CallRecord client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }}) if err != nil { t.Fatal(err) } var out map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(schema), &out) if int(count.Load()) != tc.attempts { t.Fatalf("HTTP attempts=%d, want %d", count.Load(), tc.attempts) } if tc.malformed != structuredoutput.IsMalformedError(err) { t.Fatalf("malformed classification: %v", err) } if tc.cause != nil && !errors.Is(err, tc.cause) { t.Fatalf("cause %v missing from %v", tc.cause, err) } if tc.cause == nil && !tc.malformed && err != nil { t.Fatal(err) } var failure *structuredoutput.Failure if tc.reason != "" { if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != tc.reason || !structuredoutput.IsRecoverable(err) { t.Fatalf("failure classification: %#v, %v", failure, err) } if strings.Contains(err.Error(), "provider error") || strings.Contains(err.Error(), "adapter-secret") { t.Fatalf("unsafe model error: %v", err) } } if !response.Execution.Available || response.Execution.ProviderAttempts != tc.attempts || response.Execution.StructuralRepairs != tc.repairCalls || response.Execution.TransientRetries != tc.transientRetries { t.Fatalf("returned execution counts: %+v", response.Execution) } if response.Execution.RetriesExhausted != (tc.name == "positive retry exhausted") { t.Fatalf("retry exhaustion: %+v", response.Execution) } if tc.reason == outcomes.ReasonProviderFailed && (response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusServiceUnavailable) { t.Fatalf("provider status missing: %+v", response.Execution) } if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) { t.Fatal("zero retry reported exhaustion") } if response.Prepared == nil || response.Prompt.DefinitionHash == "" || response.Prompt.RenderedHash == "" || response.Prompt.ProfileID != "base" || response.Prompt.BackendID != "offline" { t.Fatalf("prepared metadata missing: %+v", response) } if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role { t.Fatalf("logical records: %+v", records) } if tc.malformed && records[1].FinalContent == "" { t.Fatal("invalid final content lost") } if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries { t.Fatalf("execution counts: %+v", events) } if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") { t.Fatalf("decoded response: %+v, %v", response, out) } }) } } } func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) { for _, tc := range []struct { name, original, replacement string valid bool repairs int }{ {name: "empty replacement", original: "uh ", replacement: `"corrected_text":""`, valid: true, repairs: 2}, {name: "spaces replacement", original: "hello", replacement: `"corrected_text":" "`, valid: true, repairs: 2}, {name: "ordinary replacement", original: " ", replacement: `"corrected_text":" "`, valid: true, repairs: 2}, {name: "missing replacement", original: "uh ", replacement: "", valid: false}, {name: "null replacement", original: "uh ", replacement: `"corrected_text":null`, valid: false}, {name: "numeric replacement", original: "uh ", replacement: `"corrected_text":1`, valid: false}, {name: "empty original", original: "", replacement: `"corrected_text":"x"`, valid: false}, } { t.Run(tc.name, func(t *testing.T) { fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original), `"confidence":0.9`} if tc.replacement != "" { fields = append(fields, tc.replacement) } payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}` runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } var out map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) if got := count.Load(); got != 1 { t.Fatalf("expected one provider generation, got %d", got) } if response.Execution.StructuralRepairs != 0 || response.Execution.ProviderAttempts != 1 { t.Fatalf("unexpected repair/generation counts: %+v", response.Execution) } if tc.valid { if err != nil { t.Fatal(err) } corrections := out["corrections"].([]any) item := corrections[0].(map[string]any) var expected string if err := json.Unmarshal([]byte(strings.TrimPrefix(tc.replacement, `"corrected_text":`)), &expected); err != nil { t.Fatal(err) } if item["corrected_text"] != expected || item["original_text"] != tc.original { t.Fatalf("span values changed: %+v", item) } } else if !structuredoutput.IsMalformedError(err) { t.Fatalf("expected structural rejection, got %v", err) } }) } } func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { requestSeen := false var posted struct { Messages []structuredoutput.Message `json:"messages"` } runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { requestSeen = true if r.Header.Get("Authorization") != "Bearer adapter-secret" { t.Error("backend credential not used") } if err := json.NewDecoder(r.Body).Decode(&posted); err != nil { t.Error(err) } }) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } req := adapterRequest(responseschema.CorrectionSetKey) req.Variables["HardeningPolicy"] = "attacker-controlled" var out map[string]any response, err := client.CompleteStructured(context.Background(), req, &out) if err != nil { t.Fatal(err) } if !requestSeen || count.Load() != 1 || response.Prepared == nil || len(response.Prepared.Messages) == 0 { t.Fatal("prepared call missing") } if len(posted.Messages) != len(response.Prepared.Messages) { t.Fatal("prepared and dispatched message counts differ") } for i := range posted.Messages { if posted.Messages[i] != response.Prepared.Messages[i] { t.Fatal("prepared messages differ from dispatched request") } } found := false for _, msg := range response.Prepared.Messages { if strings.Contains(msg.Content, prompts.HardeningText()) { found = true } if strings.Contains(msg.Content, "attacker-controlled") { t.Fatal("reserved hardening variable was overridden") } } if !found { t.Fatal("hardening policy missing") } backend := runtime.Config.Backends["offline"] backend.Endpoint = "http://127.0.0.1:1/v1" runtime.Config.Backends["offline"] = backend if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 { t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load()) } } func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) { cases := []struct { name string system string user string replies []reply repairs int wantSystem string wantUser string }{ { name: "operator text without shared references", system: "Operator system instructions.", user: "Operator request without variables.", replies: []reply{{200, `{"wrong":true}`}, {200, `{"corrections":[]}`}}, repairs: 1, wantSystem: "Operator system instructions.", wantUser: "Operator request without variables.", }, { name: "operator text with shared references", system: "Policy: {{.HardeningPolicy}}", user: "Section: {{.SectionJSON}}", replies: []reply{{200, `{"corrections":[]}`}}, wantSystem: "Policy: " + prompts.HardeningText(), wantUser: "Section: []", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var posted []struct { Messages []structuredoutput.Message `json:"messages"` } runtime, count, _ := adapterFixtureWithPrompt(t, tc.replies, 0, tc.repairs, func(r *http.Request) { var envelope struct { Messages []structuredoutput.Message `json:"messages"` } if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil { t.Error(err) return } posted = append(posted, envelope) }, operatorPromptDocument(tc.system, tc.user)) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } req := adapterRequest(responseschema.CorrectionSetKey) if tc.name == "operator text with shared references" { req.Variables["HardeningPolicy"] = "attacker-controlled" } var out map[string]any response, err := client.CompleteStructured(context.Background(), req, &out) if err != nil { t.Fatal(err) } if int(count.Load()) != len(tc.replies) || response.Execution.StructuralRepairs != tc.repairs { t.Fatalf("schema/repair execution = %d calls, %+v; want %d calls and %d repair", count.Load(), response.Execution, len(tc.replies), tc.repairs) } if response.Prepared == nil || len(posted) == 0 || len(posted[0].Messages) != len(response.Prepared.Messages) { t.Fatal("prepared operator prompt was not dispatched") } for i := range response.Prepared.Messages { if response.Prepared.Messages[i] != posted[0].Messages[i] { t.Fatal("prepared operator messages differ from the first dispatched request") } } if len(response.Prepared.Messages) != 2 || response.Prepared.Messages[0].Content != tc.wantSystem || response.Prepared.Messages[1].Content != tc.wantUser { t.Fatalf("prepared messages = %+v", response.Prepared.Messages) } if tc.name == "operator text with shared references" { if strings.Contains(response.Prepared.Messages[0].Content, "attacker-controlled") || !strings.Contains(response.Prepared.Messages[0].Content, prompts.HardeningText()) { t.Fatal("caller variables replaced the reserved shared value") } } else if strings.Contains(response.Prepared.Messages[0].Content+response.Prepared.Messages[1].Content, prompts.HardeningText()) { t.Fatal("shared text was injected into operator messages that omit its reference") } }) } } func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) { t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used") cases := []struct { name string parentTemperature string childTemperature string wantTemperature bool }{ {name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true}, {name: "omitted", wantTemperature: false}, {name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature + " base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature + " extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n" operatorBackend := `backends: openrouter: endpoint: https://operator.invalid/v1 concurrency_limit: 91 queue_capacity: 92 api_key_env: CATALOG_TRAP_KEY extra_params: routing: inherited: backend overridden: backend cleared: remove-me ` documents := map[string]string{ "unrelated-profile-file.yml": profileDocument, "provider-tuning.yaml": operatorBackend, } var wire map[string]json.RawMessage runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" { t.Errorf("application credential was not authoritative: %q", got) } if err := json.NewDecoder(r.Body).Decode(&wire); err != nil { t.Error(err) } }, "openrouter", documents) profile := runtime.Profiles["proposal"] if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 { t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams) } client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } var out map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) if err != nil { t.Fatal(err) } if count.Load() != 1 { t.Fatalf("provider calls = %d, want 1", count.Load()) } temperature, hasTemperature := wire["temperature"] if hasTemperature != tc.wantTemperature { t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire) } if tc.wantTemperature { var value float64 if err := json.Unmarshal(temperature, &value); err != nil || value != 0 { t.Fatalf("explicit zero temperature = %s (%v)", temperature, err) } } var routing map[string]any if err := json.Unmarshal(wire["routing"], &routing); err != nil { t.Fatalf("provider routing fields missing: %s (%v)", wire, err) } if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" { t.Fatalf("recursively overlaid provider fields = %#v", routing) } if value, exists := routing["cleared"]; !exists || value != nil { t.Fatalf("null did not replace inherited provider field: %#v", routing) } if response.Model != "full-model" || response.Prepared == nil { t.Fatalf("execution did not use selected profile: %+v", response) } }) } } func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) { var posted []struct { Model string `json:"model"` Messages []structuredoutput.Message `json:"messages"` } runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { var envelope struct { Model string `json:"model"` Messages []structuredoutput.Message `json:"messages"` } if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil { t.Error(err) return } posted = append(posted, envelope) }, operatorPromptDocument("Initial operator instructions.", "Initial payload.")) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } profile := "profiles:\n base:\n backend: offline\n model: edited-model\n" if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil { t.Fatal(err) } updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.") if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil { t.Fatal(err) } var out map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) if err != nil { t.Fatal(err) } if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" { t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0]) } main := filepath.Join(dir, "config.yml") newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient) if err != nil { t.Fatal(err) } newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) if err != nil { t.Fatal(err) } if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" { t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1]) } } func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } req := adapterRequest(responseschema.CorrectionSetKey) req.PromptID = "missing.prompt" var out map[string]any if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 { t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load()) } } func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{503, ""}}, 2, 0, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() var once sync.Once client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) { if e.Kind == pk.ExecutionAttempt && e.Dispatched { once.Do(cancel) } }}) if err != nil { t.Fatal(err) } var out map[string]any _, err = client.CompleteStructured(ctx, adapterRequest(responseschema.CorrectionSetKey), &out) if !errors.Is(err, context.Canceled) || count.Load() != 1 { t.Fatalf("cancellation: %v, HTTP calls=%d", err, count.Load()) } } func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) { blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) } runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{ "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n", }) runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } var output map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) var failure *structuredoutput.Failure if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available { t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution) } parent, cancel := context.WithCancel(context.Background()) cancel() response, err = client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output) if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureCanceled || failure.Reason != outcomes.ReasonCanceled || !errors.Is(err, context.Canceled) || response.Execution.Available { t.Fatalf("parent cancellation: failure=%#v counts=%+v", failure, response.Execution) } } func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) { runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil) client, err := runtime.NewRoleClient("validation", CallHooks{}) if err != nil { t.Fatal(err) } var output struct { Validations []struct { CorrectionIndex int `json:"correction_index"` } `json:"validations"` } response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.ValidatorDecisionSetKey), &output) var failure *structuredoutput.Failure if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonInvalidOutput || !response.Execution.Available { t.Fatalf("decode overflow: failure=%#v counts=%+v", failure, response.Execution) } _, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), nil) if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { t.Fatalf("invalid target: %#v", failure) } req := adapterRequest("missing_schema") _, err = client.CompleteStructured(context.Background(), req, &output) if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { t.Fatalf("unknown schema: %#v", failure) } req = adapterRequest(responseschema.ValidatorDecisionSetKey) req.PromptID = "missing.prompt" _, err = client.CompleteStructured(context.Background(), req, &output) if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { t.Fatalf("missing prompt: %#v", failure) } } func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) } var output map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) var failure *structuredoutput.Failure if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonProviderFailed || count.Load() != 1 || response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusBadRequest { t.Fatalf("non-retryable provider failure: failure=%#v calls=%d counts=%+v", failure, count.Load(), response.Execution) } } func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) { parent, cancel := context.WithCancel(context.Background()) defer cancel() cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation) failure := classifiedFailure(parent, parent, cause, nil) if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) { t.Fatalf("unknown validation failure: %#v", failure) } }