From c8a78bf024e45b2f3ed5ec4914d93c324bdbad93 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 01:57:25 +0000 Subject: [PATCH] Use PromptKit prepared execution for structured model calls --- docs/integrations/openai-compatible-llm.md | 97 +-- docs/internal/llm-runtime.md | 84 +-- docs/roadmap/implementation.md | 10 +- internal/framework/llm/adapter.go | 138 +++++ internal/framework/llm/adapter_test.go | 257 ++++++++ internal/framework/llm/client_common.go | 33 - internal/framework/llm/effective_config.go | 55 -- .../framework/llm/effective_config_test.go | 106 ---- .../framework/llm/openai_compatible_client.go | 370 ------------ .../llm/openai_compatible_client_test.go | 571 ------------------ internal/framework/llm/secrets.go | 11 +- internal/framework/llm/secrets_test.go | 49 +- internal/framework/structuredoutput/call.go | 40 +- .../framework/structuredoutput/failure.go | 27 + .../structuredoutput/failure_test.go | 18 + .../framework/structuredoutput/malformed.go | 28 - .../structuredoutput/malformed_test.go | 32 - 17 files changed, 560 insertions(+), 1366 deletions(-) create mode 100644 internal/framework/llm/adapter.go create mode 100644 internal/framework/llm/adapter_test.go delete mode 100644 internal/framework/llm/effective_config.go delete mode 100644 internal/framework/llm/effective_config_test.go delete mode 100644 internal/framework/llm/openai_compatible_client.go delete mode 100644 internal/framework/llm/openai_compatible_client_test.go create mode 100644 internal/framework/structuredoutput/failure.go create mode 100644 internal/framework/structuredoutput/failure_test.go delete mode 100644 internal/framework/structuredoutput/malformed.go delete mode 100644 internal/framework/structuredoutput/malformed_test.go diff --git a/docs/integrations/openai-compatible-llm.md b/docs/integrations/openai-compatible-llm.md index 4f1b96c..434ca12 100644 --- a/docs/integrations/openai-compatible-llm.md +++ b/docs/integrations/openai-compatible-llm.md @@ -1,76 +1,31 @@ # OpenAI-Compatible LLM Integration -This document owns the external protocol used by Audita's structured LLM adapter. -Settings, inheritance, defaults, and credential supply belong to -[configuration](../config.md#llm-settings); implementation and scheduling belong to -[LLM runtime internals](../internal/llm-runtime.md). +Audita uses the pinned [PromptKit](https://gitea.maximumdirect.net/eric/promptkit) +built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing, +provider envelope parsing, transient retries, per-generation timeouts, and JSON +Schema repair. A backend's configured base endpoint must support the selected +profile's model and structured JSON-schema responses; local configuration +inspection does not contact the provider. -## Endpoint and Authentication +Audita selects a role profile, supplies its registered backend credential, and +passes the embedded response schema as a complete per-call output contract. It +prepares each call once, checks that the prepared target still matches the +inspected backend and endpoint, then executes that same prepared request. A +profile cannot introduce a separate credential environment lookup. An absent +backend credential permits an unauthenticated local service. -Audita sends an HTTP `POST` to the configured base URL with trailing slashes -removed and `/chat/completions` appended. Supply an HTTP or HTTPS base URL that -makes this the intended endpoint; the adapter does not require HTTPS in code. -Requests have `Content-Type: application/json`. A non-empty configured API key -adds `Authorization: Bearer `; otherwise that header is omitted. +`llm.transient_retries` counts additional attempts for eligible transient +failures; explicit zero makes one attempt. `llm.repair_attempts` counts +additional generations for schema-invalid model output. The operation timeout +covers preparation, capacity waiting, retries, generation, and repair. Caller +cancellation or an earlier deadline also applies. See [configuration](../config.md#llm-settings) +for defaults and [PromptKit's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/consumers/pkg-promptkit.md) +for the pinned protocol and retry details. -The endpoint must support the configured model and structured JSON-schema -responses. Local configuration validation does not probe that support. - -## Request Shape - -The payload contains `model`, an array of `messages` with string `role`/`content`, -and `response_format`. The structured-output portion has this illustrative shape: - -```json -{ - "type": "json_schema", - "json_schema": { - "name": "correction_set", - "strict": true, - "schema": {"type": "object"} - } -} -``` - -The actual schema is supplied for the call; the fragment above is not a complete -request schema. Runtime calls use `correction_set` and `validator_decision_set` -identities. The adapter requires a non-empty resolved model, non-empty message -roles/content, a schema name, and valid schema JSON. It sends strict mode to the -provider; local typed decoding and proposal/decision checks still apply afterward. - -## Response Handling Contract - -A successful response must be a JSON envelope with at least one `choices` item. -Audita reads the first choice's `message.content`, accepting a non-empty string -containing JSON or a non-null raw JSON value that can decode into the expected -output type. Empty content, missing choices, and incompatible payloads fail. -The adapter also reads optional `model` and token-usage metadata. - -There is no streaming protocol or provider SDK dependency in this integration. -Audita uses this chat-completions contract, not other provider API shapes. - -## Retries and Timeouts - -The effective adapter retry count is additional attempts after the initial request; -see [configuration](../config.md#llm-settings) for the current zero-value behavior. -Transport/read errors, malformed response envelopes/content, typed decode failures, -HTTP `429`, and server-error responses are retryable. Other non-success HTTP -responses and a response missing choices are not retried by the adapter. -Cancellation or an expired context stops retrying. - -The configured HTTP timeout applies per request; it is not a whole-process time -budget. A parent process can impose its own overall timeout. See -[configuration](../config.md#llm-settings) for timeout and retry values and -[subprocess guidance](subprocess.md) for parent responsibilities. - -## Error Behavior - -For non-success responses, error detail comes from provider `error.message`, then -a top-level `message`, otherwise the HTTP status and response body. Configured -keys are redacted from surfaced errors and LLM diagnostic payloads. - -Malformed structured responses can become section/batch warnings and rejected -proposals rather than fatal run errors, depending on the call site. Other adapter -failures can terminate processing with partial results. Consult the -[report contract](output-files.md#module-outcomes) for those outcomes and -[operations](../operations.md#artifact-handling) for handling diagnostic contents. +A failed JSON Schema result after repair is classified as malformed generated +output; proposal and validator callers can turn it into a warning. Provider, +transport, malformed HTTP envelope, retry, admission, and deadline failures stay +runtime failures. Audita decodes into domain types only after PromptKit validates +the JSON. Errors exposed to users carry safe messages while retaining typed +causes for internal classification. Diagnostic request/response content is +sensitive and follows [operations guidance](../operations.md#artifact-handling). diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index 937d88d..deb74f2 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -1,68 +1,34 @@ # Audita LLM Runtime -## Scope - -This document describes the structured LLM runtime and scheduler behavior. - ## Client Boundary -All runtime LLM calls go through `contracts.StructuredLLMClient`. +`llm.NewRuntime` assembles one PromptKit engine for both roles. It loads embedded +prompt and schema assets, registers application backends, and inspects the +selected proposal and validation profiles offline. A role-bound `llm.RoleClient` +implements `structuredoutput.Client`; proposal and validator contracts alias that +interface during migration. Both roles share engine-local backend capacity. -Primary adapter: -- `internal/framework/llm/OpenAICompatibleClient` +For each logical call, the adapter starts an operation context and a diagnostic +record before preparation. It supplies the reserved hardening text, the selected +profile and resolved backend credential, and the complete embedded schema/repair +contract. `PrepareExecution` produces the exact messages and hashes retained in +an Audita snapshot. The adapter checks the prepared backend and endpoint before +`RunPrepared`. It records available final content, usage, model, and profile +metadata and decodes only a validated result. Call hooks expose logical +start/finish records and PromptKit execution events for the report collector. -## Adapter and Response Schema Registry +The [integration contract](../integrations/openai-compatible-llm.md) describes +wire ownership and failure behavior. `structuredoutput.Failure` has a safe +message and a typed cause; only exhausted structural repair is a malformed +output failure. Secrets from all configured backends, including unselected +ones, are available to the diagnostics redaction boundary. -The adapter implements the [LLM wire contract](../integrations/openai-compatible-llm.md). -It maps internal structured requests to HTTP messages and decodes provider content -into the supplied Go output target. Endpoint, authentication, response shapes, and -retry conditions are defined in that contract rather than repeated here. +## Current Orchestration -`internal/framework/responseschema` stores the per-call response schema definitions -and diagnostic metadata. Proposal generation and validation choose their schema -before invoking the client. Provider strict mode does not replace local checks. +The old `llm.Scheduler` and runner instrumentation still exist pending the +orchestration migration. The final runtime uses PromptKit backend admission, +retry, and repair rather than that scheduler. `llm.DiagnosticsWriter` remains +the current protected artifact writer. -## Error Classification - -The adapter marks retryable errors internally and uses context state and attempt -count to decide whether to retry. It sanitizes errors before returning them. -`internal/framework/structuredoutput` classifies malformed output for proposal -and validator call sites, which can turn those errors into warnings and rejected -section/batch work. Other errors propagate through the runner. - -## Scheduling and Concurrency - -`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating. - -CLI orchestration composes global and stage schedulers; runner instrumentation -measures them. Limit definitions and inheritance belong to -[configuration](../config.md#concurrency). - -Scheduler release is guarded to avoid permit leaks on cancellation/error. Runner -collects utilization observations; their serialized shape is defined by the -[output-file contract](../integrations/output-files.md#utilization-diagnostics). - -## Diagnostics and Redaction - -`internal/framework/llm/DiagnosticsWriter` writes request/response/error artifacts. - -Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted from: -- diagnostics payloads; -- surfaced runtime/adapter errors. - -## Changing LLM Integration or Scheduling - -Review client, scheduler, redaction, and structured-output malformed-response -coverage alongside runner orchestration and utilization tests. Use the -[testing policy](../policy/testing.md) to select sufficient checks; preserve the -[LLM integration contract](../integrations/openai-compatible-llm.md) and -[architecture invariants](../policy/architecture.md#correction-and-concurrency-invariants). - -## Key Tests - -- `internal/framework/llm/openai_compatible_client_test.go` -- `internal/framework/llm/scheduler_test.go` -- `internal/framework/llm/diagnostics_test.go` -- `internal/framework/responseschema/registry_test.go` -- `internal/framework/structuredoutput/malformed_test.go` -- `internal/framework/runner/*_test.go` (orchestration and utilization) +Focused adapter tests use controlled local HTTP responses. Complete runner and +CLI integration checks follow the caller and orchestration migration. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 063f6f6..0a7e731 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -264,7 +264,15 @@ secret absence and overlay/CLI provenance, including inherited validation select ## Stage 5: Implement the Final PromptKit Structured-Call Adapter -**Status:** Pending. **Depends on:** Stages 3–4. +**Status:** Complete (2026-09-24). Added role-bound prepared execution with +target verification, validated-only decoding, typed safe failures, protected +prepared/final-content snapshots, and logical-call and execution-event hooks. +Removed the old direct HTTP client and role resolver; updated LLM integration +documentation. `go test ./internal/framework/llm` and +`go test ./internal/framework/structuredoutput` passed, including controlled +HTTP repair/retry, target-drift, transport/protocol, and cancellation cases; +`git diff --check` passed. Caller integration remains for Stages 6–8. +**Depends on:** Stages 3–4. **Owns:** `internal/framework/llm`, the typed failure boundary in `internal/framework/structuredoutput`, and adapter tests. diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go new file mode 100644 index 0000000..ccde461 --- /dev/null +++ b/internal/framework/llm/adapter.go @@ -0,0 +1,138 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" + "gitea.maximumdirect.net/eric/audita/internal/prompts" + pk "gitea.maximumdirect.net/eric/promptkit" +) + +// CallHooks receive one logical start and finish record and the execution +// events for this call. Call records can contain transcript content. +type CallHooks struct { + OnCall func(structuredoutput.CallRecord) + OnEvent pk.ExecutionObserver +} + +type RoleClient struct { + runtime *Runtime + role string + hooks CallHooks +} + +var _ structuredoutput.Client = (*RoleClient)(nil) + +func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, error) { + if _, ok := r.Profiles[role]; !ok { + return nil, fmt.Errorf("unknown LLM role %q", role) + } + return &RoleClient{runtime: r, role: role, hooks: hooks}, nil +} + +func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) { + ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout) + defer cancel() + record := structuredoutput.CallRecord{Context: req.Context, Started: time.Now().UTC()} + record.Context.Role = c.role + if c.hooks.OnCall != nil { + c.hooks.OnCall(record) + } + defer func() { + record.Finished = time.Now().UTC() + if err != nil { + if structuredoutput.IsMalformedError(err) { + record.Failure = structuredoutput.FailureMalformedOutput + } else { + record.Failure = structuredoutput.FailureRuntime + } + } + if c.hooks.OnCall != nil { + c.hooks.OnCall(record) + } + }() + if err = validateOutputTarget(out); err != nil { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "invalid structured output target", Cause: err} + } + schemaPath, ok := responseschema.AssetPath(req.Schema) + if !ok { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)} + } + profile := c.runtime.Profiles[c.role] + backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID] + vars := make(map[string]string, len(req.Variables)+1) + for key, value := range req.Variables { + vars[key] = value + } + vars["HardeningPolicy"] = prompts.HardeningText() + apiKey := "" + if backend.Credential != nil { + apiKey = backend.Credential.Value() + } + handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{ + PromptID: req.PromptID, PromptVersion: req.PromptVersion, + ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars, + Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts}, + Observer: c.hooks.OnEvent, + }) + if prepErr != nil { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "could not prepare structured model call", Cause: prepErr} + } + defer handle.Discard() + details := handle.Details() + response.Prompt = promptMetadata(details) + response.Model = details.EffectiveModelParams.Model + response.Prepared = preparedSnapshot(details) + record.Prepared = response.Prepared + selected, exists := c.runtime.Config.Backends[details.SelectedBackendID] + endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") } + if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")} + } + result, runErr := c.runtime.Engine.RunPrepared(ctx, handle) + if result != nil { + response.FinalContent = result.RawOutput + record.FinalContent = result.RawOutput + response.Model = result.ModelName + response.PromptTokens = result.Usage.PromptTokens + response.CompletionTokens = result.Usage.CompletionTokens + response.TotalTokens = result.Usage.TotalTokens + } + if runErr != nil { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: runErr} + } + if result == nil { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call returned no result"} + } + if !result.Validation.IsValid { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"} + } + if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil { + return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "validated structured output could not be decoded", Cause: decodeErr} + } + response.Content = json.RawMessage(result.RawOutput) + response.Provider = "openai-compatible" + return response, nil +} + +func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata { + return structuredoutput.PromptMetadata{ + PromptID: details.PromptID, PromptVersion: details.PromptVersion, + DefinitionHash: details.PromptHash, RenderedHash: details.RenderedPromptHash, + ProfileID: details.SelectedProfileID, BackendID: details.SelectedBackendID, + Model: details.EffectiveModelParams.Model, + } +} + +func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot { + messages := make([]structuredoutput.Message, len(details.Messages)) + for i, message := range details.Messages { + messages[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content} + } + return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)} +} diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go new file mode 100644 index 0000000..c900033 --- /dev/null +++ b/internal/framework/llm/adapter_test.go @@ -0,0 +1,257 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + ck "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" + "gitea.maximumdirect.net/eric/audita/internal/prompts" + pk "gitea.maximumdirect.net/eric/promptkit" +) + +type reply struct { + status int + content string +} + +func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *int, string) { + t.Helper() + count := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count++ + if hook != nil { + hook(r) + } + step := replies[len(replies)-1] + if count <= len(replies) { + step = replies[count-1] + } + if step.status == 0 { + connection, _, hijackErr := w.(http.Hijacker).Hijack() + if hijackErr != nil { + t.Error(hijackErr) + return + } + _ = connection.Close() + return + } + if step.status != http.StatusOK { + w.WriteHeader(step.status) + _, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`) + return + } + if step.content == "MALFORMED_ENVELOPE" { + _, _ = fmt.Fprint(w, `{"choices":[]}`) + return + } + encoded, _ := json.Marshal(step.content) + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) + })) + t.Cleanup(server.Close) + t.Setenv("ADAPTER_KEY", "adapter-secret") + base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs) + dir := profileLayout(t, base, "") + main := filepath.Join(dir, "config.yml") + profile := "id: base\nbackend: offline\nmodel: full-model\n" + if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.File(main), "") + if err != nil { + t.Fatal(err) + } + runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client()) + if err != nil { + t.Fatal(err) + } + return runtime, &count, dir +} + +func adapterRequest(schema responseschema.Key) structuredoutput.Request { + id := prompts.PromptIDModuleGlossaryProposal + vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"} + if schema == responseschema.ValidatorDecisionSetKey { + id = prompts.PromptIDValidatorEditorialReview + vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"} + } + return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} +} + +func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { + for _, schema := range []responseschema.Key{responseschema.CorrectionSetKey, responseschema.ValidatorDecisionSetKey} { + valid := `{"corrections":[]}` + if schema == responseschema.ValidatorDecisionSetKey { + valid = `{"validations":[]}` + } + for _, tc := range []struct { + name string + replies []reply + retries, repairs, attempts, repairCalls, transientRetries int + malformed bool + cause error + }{ + {"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, nil}, + {"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, nil}, + {"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, nil}, + {"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil}, + {"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil}, + {"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, pk.ErrLLMGenerate}, + {"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, pk.ErrRetriesExhausted}, + {"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, pk.ErrTransport}, + {"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, pk.ErrMalformedResponse}, + {"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, nil}, + } { + t.Run(string(schema)+"/"+tc.name, func(t *testing.T) { + runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil) + role := "proposal" + if schema == responseschema.ValidatorDecisionSetKey { + role = "validation" + } + var events []pk.ExecutionEvent + var records []structuredoutput.CallRecord + client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }}) + if err != nil { + t.Fatal(err) + } + var out map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(schema), &out) + if *count != tc.attempts { + t.Fatalf("HTTP attempts=%d, want %d", *count, tc.attempts) + } + if tc.malformed != structuredoutput.IsMalformedError(err) { + t.Fatalf("malformed classification: %v", err) + } + if tc.cause != nil && !errors.Is(err, tc.cause) { + t.Fatalf("cause %v missing from %v", tc.cause, err) + } + if tc.cause == nil && !tc.malformed && err != nil { + t.Fatal(err) + } + if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) { + t.Fatal("zero retry reported exhaustion") + } + if response.Prepared == nil || response.Prompt.DefinitionHash == "" || response.Prompt.RenderedHash == "" || response.Prompt.ProfileID != "base" || response.Prompt.BackendID != "offline" { + t.Fatalf("prepared metadata missing: %+v", response) + } + if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role { + t.Fatalf("logical records: %+v", records) + } + if tc.malformed && records[1].FinalContent == "" { + t.Fatal("invalid final content lost") + } + if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries { + t.Fatalf("execution counts: %+v", events) + } + if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") { + t.Fatalf("decoded response: %+v, %v", response, out) + } + }) + } + } +} + +func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { + requestSeen := false + var posted struct { + Messages []structuredoutput.Message `json:"messages"` + } + runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { + requestSeen = true + if r.Header.Get("Authorization") != "Bearer adapter-secret" { + t.Error("backend credential not used") + } + if err := json.NewDecoder(r.Body).Decode(&posted); err != nil { + t.Error(err) + } + }) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + req := adapterRequest(responseschema.CorrectionSetKey) + req.Variables["HardeningPolicy"] = "attacker-controlled" + var out map[string]any + response, err := client.CompleteStructured(context.Background(), req, &out) + if err != nil { + t.Fatal(err) + } + if !requestSeen || *count != 1 || response.Prepared == nil || len(response.Prepared.Messages) == 0 { + t.Fatal("prepared call missing") + } + if len(posted.Messages) != len(response.Prepared.Messages) { + t.Fatal("prepared and dispatched message counts differ") + } + for i := range posted.Messages { + if posted.Messages[i] != response.Prepared.Messages[i] { + t.Fatal("prepared messages differ from dispatched request") + } + } + found := false + for _, msg := range response.Prepared.Messages { + if strings.Contains(msg.Content, prompts.HardeningText()) { + found = true + } + if strings.Contains(msg.Content, "attacker-controlled") { + t.Fatal("reserved hardening variable was overridden") + } + } + if !found { + t.Fatal("hardening policy missing") + } +} + +func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) { + runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + req := adapterRequest(responseschema.CorrectionSetKey) + req.PromptID = "missing.prompt" + var out map[string]any + if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || *count != 0 { + t.Fatalf("preparation failure dispatched: %v, %d", err, *count) + } + // PrepareExecution reloads the profile, so a changed target must be blocked. + if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nendpoint: http://127.0.0.1:1/v1\nmodel: full-model\n"), 0600); err != nil { + t.Fatal(err) + } + req = adapterRequest(responseschema.CorrectionSetKey) + _, err = client.CompleteStructured(context.Background(), req, &out) + if err == nil || !strings.Contains(err.Error(), "target changed") || *count != 0 { + t.Fatalf("target drift dispatched: %v, %d", err, *count) + } +} + +func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { + runtime, count, _ := adapterFixture(t, []reply{{503, ""}}, 2, 0, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var once sync.Once + client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) { + if e.Kind == pk.ExecutionAttempt && e.Dispatched { + once.Do(cancel) + } + }}) + if err != nil { + t.Fatal(err) + } + var out map[string]any + _, err = client.CompleteStructured(ctx, adapterRequest(responseschema.CorrectionSetKey), &out) + if !errors.Is(err, context.Canceled) || *count != 1 { + t.Fatalf("cancellation: %v, HTTP calls=%d", err, *count) + } +} diff --git a/internal/framework/llm/client_common.go b/internal/framework/llm/client_common.go index 5526c8f..b4fd8fb 100644 --- a/internal/framework/llm/client_common.go +++ b/internal/framework/llm/client_common.go @@ -2,29 +2,9 @@ package llm import ( "fmt" - "net/http" "reflect" - "strings" - "time" ) -func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client { - if base == nil { - if timeout <= 0 { - return http.DefaultClient - } - return &http.Client{Timeout: timeout} - } - - if timeout <= 0 { - return base - } - - cloned := *base - cloned.Timeout = timeout - return &cloned -} - func validateOutputTarget(out any) error { if out == nil { return fmt.Errorf("output target must not be nil") @@ -35,16 +15,3 @@ func validateOutputTarget(out any) error { } return nil } - -func sanitizeError(err error, apiKey string) error { - if err == nil { - return nil - } - msg := err.Error() - key := strings.TrimSpace(apiKey) - if key != "" { - msg = strings.ReplaceAll(msg, key, "[REDACTED]") - msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]") - } - return fmt.Errorf("%s", msg) -} diff --git a/internal/framework/llm/effective_config.go b/internal/framework/llm/effective_config.go deleted file mode 100644 index e45ac91..0000000 --- a/internal/framework/llm/effective_config.go +++ /dev/null @@ -1,55 +0,0 @@ -package llm - -import ( - "net/http" - "strings" - "time" - - "gitea.maximumdirect.net/eric/audita/internal/core/config" -) - -// EffectiveConfig is the runtime LLM configuration consumed by adapters and -// schedulers after primary/validation inheritance is resolved. -type EffectiveConfig struct { - APIKey string - Model string - BaseURL string - RequestTimeout time.Duration - MaxRetries int - Concurrency int -} - -// ResolvePrimaryConfig resolves the primary LLM settings into runtime form. -func ResolvePrimaryConfig(cfg config.Config) EffectiveConfig { - return resolveFromLLMConfig(cfg.PrimaryLLM, cfg.TotalLLMConcurrency) -} - -// ResolveValidationConfig resolves validation LLM settings with inheritance -// from primary values when validation overrides are unset. -func ResolveValidationConfig(cfg config.Config) EffectiveConfig { - return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency()) -} - -// ToOpenAICompatibleClientConfig converts an effective runtime config into -// direct HTTP adapter config. -func (c EffectiveConfig) ToOpenAICompatibleClientConfig(httpClient *http.Client) OpenAICompatibleClientConfig { - return OpenAICompatibleClientConfig{ - BaseURL: c.BaseURL, - Model: c.Model, - APIKey: c.APIKey, - MaxRetries: c.MaxRetries, - HTTPClient: httpClient, - RequestTimeout: c.RequestTimeout, - } -} - -func resolveFromLLMConfig(raw config.LLMConfig, concurrency int) EffectiveConfig { - return EffectiveConfig{ - APIKey: strings.TrimSpace(raw.APIKey), - Model: strings.TrimSpace(raw.Model), - BaseURL: strings.TrimSpace(raw.BaseURL), - RequestTimeout: time.Duration(raw.TimeoutSeconds) * time.Second, - MaxRetries: raw.MaxRetries, - Concurrency: concurrency, - } -} diff --git a/internal/framework/llm/effective_config_test.go b/internal/framework/llm/effective_config_test.go deleted file mode 100644 index 84a8d39..0000000 --- a/internal/framework/llm/effective_config_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package llm - -import ( - "testing" - "time" - - "gitea.maximumdirect.net/eric/audita/internal/core/config" -) - -func TestResolvePrimaryConfig(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = " primary-key " - cfg.PrimaryLLM.Model = " model-x " - cfg.PrimaryLLM.BaseURL = " https://example.test/v1 " - cfg.PrimaryLLM.TimeoutSeconds = 42 - cfg.PrimaryLLM.MaxRetries = 7 - cfg.TotalLLMConcurrency = 3 - cfg.ProposalLLMConcurrency = 3 - - effective := ResolvePrimaryConfig(cfg) - if effective.APIKey != "primary-key" { - t.Fatalf("unexpected api key: %q", effective.APIKey) - } - if effective.Model != "model-x" { - t.Fatalf("unexpected model: %q", effective.Model) - } - if effective.BaseURL != "https://example.test/v1" { - t.Fatalf("unexpected base url: %q", effective.BaseURL) - } - if effective.RequestTimeout != 42*time.Second { - t.Fatalf("unexpected timeout: %s", effective.RequestTimeout) - } - if effective.MaxRetries != 7 { - t.Fatalf("unexpected max retries: %d", effective.MaxRetries) - } - if effective.Concurrency != 3 { - t.Fatalf("unexpected concurrency: %d", effective.Concurrency) - } -} - -func TestResolveValidationConfigInheritsPrimaryWhenUnset(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = "primary-key" - cfg.PrimaryLLM.Model = "primary-model" - cfg.PrimaryLLM.BaseURL = "https://primary.example/v1" - cfg.PrimaryLLM.TimeoutSeconds = 90 - cfg.PrimaryLLM.MaxRetries = 4 - cfg.TotalLLMConcurrency = 2 - cfg.ProposalLLMConcurrency = 2 - - effective := ResolveValidationConfig(cfg) - if effective.APIKey != "primary-key" || - effective.Model != "primary-model" || - effective.BaseURL != "https://primary.example/v1" || - effective.RequestTimeout != 90*time.Second || - effective.MaxRetries != 4 || - effective.Concurrency != 2 { - t.Fatalf("unexpected inherited config: %+v", effective) - } -} - -func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = "primary-key" - cfg.PrimaryLLM.Model = "primary-model" - cfg.PrimaryLLM.BaseURL = "https://primary.example/v1" - cfg.PrimaryLLM.TimeoutSeconds = 90 - cfg.PrimaryLLM.MaxRetries = 4 - cfg.TotalLLMConcurrency = 2 - cfg.ProposalLLMConcurrency = 2 - - timeout := 12 - retries := 9 - concurrency := 6 - cfg.ValidationLLM.APIKey = "validation-key" - cfg.ValidationLLM.Model = "validation-model" - cfg.ValidationLLM.BaseURL = "https://validation.example/v1" - cfg.ValidationLLM.TimeoutSeconds = &timeout - cfg.ValidationLLM.MaxRetries = &retries - cfg.ValidationLLMConcurrency = &concurrency - - effective := ResolveValidationConfig(cfg) - if effective.APIKey != "validation-key" || - effective.Model != "validation-model" || - effective.BaseURL != "https://validation.example/v1" || - effective.RequestTimeout != 12*time.Second || - effective.MaxRetries != 9 || - effective.Concurrency != 6 { - t.Fatalf("unexpected override config: %+v", effective) - } -} - -func TestResolveConfigOptionalAPIKey(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = "" - cfg.ValidationLLM.APIKey = "" - - primary := ResolvePrimaryConfig(cfg) - validation := ResolveValidationConfig(cfg) - if primary.APIKey != "" { - t.Fatalf("expected empty primary api key, got %q", primary.APIKey) - } - if validation.APIKey != "" { - t.Fatalf("expected empty validation api key, got %q", validation.APIKey) - } -} diff --git a/internal/framework/llm/openai_compatible_client.go b/internal/framework/llm/openai_compatible_client.go deleted file mode 100644 index 97188cd..0000000 --- a/internal/framework/llm/openai_compatible_client.go +++ /dev/null @@ -1,370 +0,0 @@ -package llm - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - "time" - - "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" -) - -const defaultOpenAICompatibleMaxRetries = 3 - -// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter. -type OpenAICompatibleClientConfig struct { - BaseURL string - Model string - APIKey string - MaxRetries int - HTTPClient *http.Client - RequestTimeout time.Duration -} - -// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with -// response_format.type=json_schema. -type OpenAICompatibleClient struct { - cfg OpenAICompatibleClientConfig - httpClient *http.Client -} - -var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil) - -func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) { - normalized, err := normalizeOpenAICompatibleConfig(cfg) - if err != nil { - return nil, err - } - return &OpenAICompatibleClient{ - cfg: normalized, - httpClient: resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout), - }, nil -} - -func (c *OpenAICompatibleClient) CompleteStructured( - ctx context.Context, - req contracts.StructuredCompletionRequest, - out any, -) (contracts.StructuredCompletionResponse, error) { - if err := validateOutputTarget(out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - - model := strings.TrimSpace(req.Model) - if model == "" { - model = c.cfg.Model - } - if model == "" { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty") - } - if req.ResponseSchema == nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema is required") - } - if strings.TrimSpace(req.ResponseSchema.Name) == "" { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty") - } - if len(req.ResponseSchema.JSONSchema) == 0 || !json.Valid(req.ResponseSchema.JSONSchema) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid") - } - - messages, err := toOpenAICompatibleMessages(req.Messages) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - - endpoint := buildChatCompletionsURL(c.cfg.BaseURL) - - var lastErr error - for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ { - content, metadata, callErr := c.completeStructuredOnce( - ctx, - endpoint, - model, - messages, - req.ResponseSchema.Name, - req.ResponseSchema.JSONSchema, - ) - if callErr == nil { - if decodeErr := json.Unmarshal(content, out); decodeErr != nil { - callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)} - } else { - return contracts.StructuredCompletionResponse{ - Content: content, - Provider: "openai-compatible", - Model: firstNonEmpty(metadata.Model, model), - PromptTokens: metadata.PromptTokens, - CompletionTokens: metadata.CompletionTokens, - TotalTokens: metadata.TotalTokens, - }, nil - } - } - - lastErr = sanitizeError(callErr, c.cfg.APIKey) - if !canRetryFromError(ctx, attempt, c.cfg.MaxRetries, callErr) { - return contracts.StructuredCompletionResponse{}, lastErr - } - } - - if lastErr == nil { - lastErr = fmt.Errorf("structured completion failed") - } - return contracts.StructuredCompletionResponse{}, lastErr -} - -type openAICompatibleMessage struct { - Role string `json:"role"` - Content string `json:"content"` -} - -type openAICompatibleRequest struct { - Model string `json:"model"` - Messages []openAICompatibleMessage `json:"messages"` - ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"` -} - -type openAICompatibleStructuredOutputShape struct { - Type string `json:"type"` - JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"` -} - -type openAICompatibleSchemaEnvelope struct { - Name string `json:"name"` - Strict bool `json:"strict"` - Schema json.RawMessage `json:"schema"` -} - -type openAICompatibleChatCompletionsResponse struct { - Model string `json:"model"` - Choices []struct { - Message struct { - Content json.RawMessage `json:"content"` - } `json:"message"` - } `json:"choices"` - Usage *openAICompatibleUsage `json:"usage,omitempty"` -} - -type openAICompatibleUsage struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` -} - -type openAICompatibleResponseMetadata struct { - Model string - PromptTokens int - CompletionTokens int - TotalTokens int -} - -func (c *OpenAICompatibleClient) completeStructuredOnce( - ctx context.Context, - endpoint string, - model string, - messages []openAICompatibleMessage, - responseSchemaName string, - responseSchemaJSON json.RawMessage, -) (json.RawMessage, openAICompatibleResponseMetadata, error) { - requestBody := openAICompatibleRequest{ - Model: model, - Messages: messages, - ResponseFormat: openAICompatibleStructuredOutputShape{ - Type: "json_schema", - JSONSchema: openAICompatibleSchemaEnvelope{ - Name: responseSchemaName, - Strict: true, - Schema: responseSchemaJSON, - }, - }, - } - - payload, err := json.Marshal(requestBody) - if err != nil { - return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err) - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) - if err != nil { - return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - if c.cfg.APIKey != "" { - httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) - } - - httpResp, err := c.httpClient.Do(httpReq) - if err != nil { - return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)} - } - defer func() { - _ = httpResp.Body.Close() - }() - - rawResp, err := io.ReadAll(httpResp.Body) - if err != nil { - return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)} - } - - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp) - if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 { - return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr} - } - return nil, openAICompatibleResponseMetadata{}, statusErr - } - - content, metadata, decodeErr := decodeChatCompletionsResponse(rawResp) - if decodeErr != nil { - return nil, openAICompatibleResponseMetadata{}, decodeErr - } - return content, metadata, nil -} - -func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) { - result := make([]openAICompatibleMessage, len(messages)) - for i, message := range messages { - role := strings.TrimSpace(message.Role) - content := strings.TrimSpace(message.Content) - if role == "" { - return nil, fmt.Errorf("message[%d] role must not be empty", i) - } - if content == "" { - return nil, fmt.Errorf("message[%d] content must not be empty", i) - } - result[i] = openAICompatibleMessage{ - Role: role, - Content: content, - } - } - return result, nil -} - -func buildChatCompletionsURL(baseURL string) string { - return strings.TrimRight(baseURL, "/") + "/chat/completions" -} - -func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) { - cfg.BaseURL = strings.TrimSpace(cfg.BaseURL) - cfg.Model = strings.TrimSpace(cfg.Model) - cfg.APIKey = strings.TrimSpace(cfg.APIKey) - if cfg.MaxRetries == 0 { - cfg.MaxRetries = defaultOpenAICompatibleMaxRetries - } - if cfg.MaxRetries < 0 { - return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater") - } - if cfg.BaseURL == "" { - return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty") - } - if cfg.Model == "" { - return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty") - } - return cfg, nil -} - -type retryableError struct { - err error -} - -func (e retryableError) Error() string { - if e.err == nil { - return "" - } - return e.err.Error() -} - -func (e retryableError) Unwrap() error { - return e.err -} - -func canRetryFromError(ctx context.Context, attempt int, maxRetries int, err error) bool { - if attempt >= maxRetries { - return false - } - if ctx.Err() != nil { - return false - } - var retryable retryableError - return errors.As(err, &retryable) -} - -func firstNonEmpty(values ...string) string { - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - return value - } - } - return "" -} - -func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) { - var parsed openAICompatibleChatCompletionsResponse - if err := json.Unmarshal(raw, &parsed); err != nil { - return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)} - } - if len(parsed.Choices) == 0 { - return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("provider response missing choices") - } - - content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content) - if err != nil { - return nil, openAICompatibleResponseMetadata{}, retryableError{err: err} - } - - meta := openAICompatibleResponseMetadata{ - Model: parsed.Model, - } - if parsed.Usage != nil { - meta.PromptTokens = parsed.Usage.PromptTokens - meta.CompletionTokens = parsed.Usage.CompletionTokens - meta.TotalTokens = parsed.Usage.TotalTokens - } - return content, meta, nil -} - -func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) { - if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" { - return nil, fmt.Errorf("provider response missing assistant message content") - } - - var textContent string - if err := json.Unmarshal(raw, &textContent); err == nil { - textContent = strings.TrimSpace(textContent) - if textContent == "" { - return nil, fmt.Errorf("provider response assistant message content is empty") - } - return json.RawMessage(textContent), nil - } - - trimmed := bytes.TrimSpace(raw) - if json.Valid(trimmed) { - return append(json.RawMessage(nil), trimmed...), nil - } - return nil, fmt.Errorf("provider response assistant message content is not valid JSON") -} - -func parseProviderErrorBody(status int, body []byte) error { - trimmed := strings.TrimSpace(string(body)) - if trimmed == "" { - return fmt.Errorf("provider returned status %d", status) - } - - var payload map[string]any - if err := json.Unmarshal(body, &payload); err == nil { - if nested, ok := payload["error"].(map[string]any); ok { - if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" { - return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg)) - } - } - if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" { - return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg)) - } - } - - return fmt.Errorf("provider returned status %d: %s", status, trimmed) -} diff --git a/internal/framework/llm/openai_compatible_client_test.go b/internal/framework/llm/openai_compatible_client_test.go deleted file mode 100644 index 9c763a6..0000000 --- a/internal/framework/llm/openai_compatible_client_test.go +++ /dev/null @@ -1,571 +0,0 @@ -package llm - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" - "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" -) - -func TestNewOpenAICompatibleClientValidation(t *testing.T) { - _, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: " ", - Model: "model", - }) - if err == nil || !strings.Contains(err.Error(), "base URL") { - t.Fatalf("expected base URL validation error, got %v", err) - } - - _, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: "https://example.test/v1", - Model: " ", - }) - if err == nil || !strings.Contains(err.Error(), "model") { - t.Fatalf("expected model validation error, got %v", err) - } - - _, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: "https://example.test/v1", - Model: "model", - MaxRetries: -1, - }) - if err == nil || !strings.Contains(err.Error(), "max retries") { - t.Fatalf("expected max retries validation error, got %v", err) - } -} - -func TestOpenAICompatibleClientRequestShapeAndDecode(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - var seenPath string - var seenAuthorization string - var seenReq map[string]any - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - seenPath = r.URL.Path - seenAuthorization = r.Header.Get("Authorization") - if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil { - t.Fatalf("decode request: %v", err) - } - - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "model":"provider-model", - "choices":[{"message":{"content":"{\"name\":\"Robby\",\"age\":22}"}}], - "usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18} -}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL + "/v1", - Model: "test-model", - APIKey: "secret-key", - MaxRetries: 1, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - type person struct { - Name string `json:"name"` - Age int `json:"age"` - } - var out person - resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - - if seenPath != "/v1/chat/completions" { - t.Fatalf("unexpected request path: %q", seenPath) - } - if seenAuthorization != "Bearer secret-key" { - t.Fatalf("unexpected authorization header: %q", seenAuthorization) - } - - responseFormat, ok := seenReq["response_format"].(map[string]any) - if !ok { - t.Fatalf("expected response_format object, got %T", seenReq["response_format"]) - } - if responseFormat["type"] != "json_schema" { - t.Fatalf("unexpected response_format.type: %v", responseFormat["type"]) - } - jsonSchema, ok := responseFormat["json_schema"].(map[string]any) - if !ok { - t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"]) - } - if jsonSchema["name"] != schema.Name { - t.Fatalf("unexpected response schema name: %v", jsonSchema["name"]) - } - if jsonSchema["strict"] != true { - t.Fatalf("expected strict=true, got %v", jsonSchema["strict"]) - } - if _, ok := jsonSchema["schema"].(map[string]any); !ok { - t.Fatalf("expected embedded JSON schema object, got %T", jsonSchema["schema"]) - } - - if out.Name != "Robby" || out.Age != 22 { - t.Fatalf("unexpected decoded output: %+v", out) - } - if resp.Provider != "openai-compatible" { - t.Fatalf("unexpected provider metadata: %q", resp.Provider) - } - if resp.Model != "provider-model" { - t.Fatalf("unexpected model metadata: %q", resp.Model) - } - if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { - t.Fatalf("unexpected token metadata: %+v", resp) - } -} - -func TestOpenAICompatibleClientDecodesCorrectionSetStructuredResponse(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9}]}"}}] -}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - type correction struct { - TargetSegmentID int `json:"id"` - OriginalText string `json:"original_text"` - CorrectedText string `json:"corrected_text"` - Confidence float64 `json:"confidence"` - } - type correctionSet struct { - Corrections []correction `json:"corrections"` - } - var out correctionSet - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - if len(out.Corrections) != 1 { - t.Fatalf("expected one correction, got %+v", out.Corrections) - } - if out.Corrections[0].TargetSegmentID != 1 || out.Corrections[0].CorrectedText != "the" { - t.Fatalf("unexpected correction payload: %+v", out.Corrections[0]) - } -} - -func TestOpenAICompatibleClientDecodesValidatorDecisionSetStructuredResponse(t *testing.T) { - schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices":[{"message":{"content":"{\"validations\":[{\"correction_index\":0,\"approved\":true,\"confidence\":0.95,\"reason\":\"ok\"}]}"}}] -}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - type validationDecision struct { - CorrectionIndex int `json:"correction_index"` - Approved bool `json:"approved"` - Confidence float64 `json:"confidence"` - Reason string `json:"reason"` - } - type validationResponse struct { - Validations []validationDecision `json:"validations"` - } - var out validationResponse - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - if len(out.Validations) != 1 { - t.Fatalf("expected one validation decision, got %+v", out.Validations) - } - if out.Validations[0].CorrectionIndex != 0 || !out.Validations[0].Approved { - t.Fatalf("unexpected validation payload: %+v", out.Validations[0]) - } -} - -func TestOpenAICompatibleClientNoAuthorizationHeaderWithoutAPIKey(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - var seenAuthorization string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - seenAuthorization = r.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - if seenAuthorization != "" { - t.Fatalf("expected empty Authorization header, got %q", seenAuthorization) - } -} - -func TestOpenAICompatibleClientMalformedJSONFailsSafely(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - MaxRetries: 0, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil || !strings.Contains(err.Error(), "decode structured output") { - t.Fatalf("expected decode error, got %v", err) - } -} - -func TestOpenAICompatibleClientMissingRequiredFieldsFailsSafely(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"model":"x","choices":[]}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - MaxRetries: 0, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil || !strings.Contains(err.Error(), "missing choices") { - t.Fatalf("expected missing-field error, got %v", err) - } -} - -func TestOpenAICompatibleClientUnknownExtraFieldsFollowLocalDecoderPolicy(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9,\"extra\":\"ignored\"}],\"top_extra\":true}"}}] -}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - type correction struct { - TargetSegmentID int `json:"id"` - OriginalText string `json:"original_text"` - CorrectedText string `json:"corrected_text"` - Confidence float64 `json:"confidence"` - } - type correctionSet struct { - Corrections []correction `json:"corrections"` - } - var out correctionSet - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("expected unknown extra fields to be ignored by local decoder, got %v", err) - } - if len(out.Corrections) != 1 || out.Corrections[0].CorrectedText != "the" { - t.Fatalf("unexpected decoded payload: %+v", out.Corrections) - } -} - -func TestOpenAICompatibleClientProviderErrorRedactsSecret(t *testing.T) { - secret := "super-secret-key" - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - _, _ = w.Write([]byte(`{"error":{"message":"Authorization failed for Bearer super-secret-key"}}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - APIKey: secret, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil { - t.Fatalf("expected provider error") - } - if strings.Contains(err.Error(), secret) { - t.Fatalf("error leaked secret: %v", err) - } - if !strings.Contains(err.Error(), "[REDACTED]") { - t.Fatalf("expected redaction marker in error: %v", err) - } -} - -func TestOpenAICompatibleClientRequestErrorRedactsSecret(t *testing.T) { - secret := "super-secret-key" - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: "https://example.test/v1", - Model: "test-model", - APIKey: secret, - HTTPClient: &http.Client{ - Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { - _ = r - return nil, fmt.Errorf("request failed for Authorization: Bearer %s", secret) - }), - }, - MaxRetries: 0, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil { - t.Fatalf("expected request error") - } - if strings.Contains(err.Error(), secret) { - t.Fatalf("error leaked secret: %v", err) - } - if !strings.Contains(err.Error(), "[REDACTED]") { - t.Fatalf("expected redaction marker in error: %v", err) - } -} - -func TestOpenAICompatibleClientCancellationAndTimeout(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - select { - case <-r.Context().Done(): - return - case <-time.After(200 * time.Millisecond): - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)) - } - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - RequestTimeout: 20 * time.Millisecond, - MaxRetries: 0, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil { - t.Fatalf("expected timeout-related error") - } - if !strings.Contains(strings.ToLower(err.Error()), "context deadline") { - t.Fatalf("expected context deadline in error, got %v", err) - } -} - -func TestOpenAICompatibleClientRetryBehavior(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - var attempts int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - current := atomic.AddInt32(&attempts, 1) - if current == 1 { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error":{"message":"temporary failure"}}`)) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - MaxRetries: 1, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - if atomic.LoadInt32(&attempts) != 2 { - t.Fatalf("expected 2 attempts, got %d", attempts) - } -} - -func TestOpenAICompatibleClientRetryOnMalformedStructuredOutput(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - var attempts int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - current := atomic.AddInt32(&attempts, 1) - w.Header().Set("Content-Type", "application/json") - if current == 1 { - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`)) - return - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)) - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - MaxRetries: 1, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured: %v", err) - } - if atomic.LoadInt32(&attempts) != 2 { - t.Fatalf("expected 2 attempts, got %d", attempts) - } -} - -func TestOpenAICompatibleClientHonorsCancelledContextWithoutRetry(t *testing.T) { - schema := responseschema.MustLookup(responseschema.CorrectionSetKey) - var attempts int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt32(&attempts, 1) - <-r.Context().Done() - })) - defer server.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{ - BaseURL: server.URL, - Model: "test-model", - MaxRetries: 3, - }) - if err != nil { - t.Fatalf("NewOpenAICompatibleClient: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - var out map[string]any - _, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}}, - ResponseSchema: &schema, - }, &out) - if err == nil { - t.Fatalf("expected cancellation error") - } - if !errors.Is(err, context.Canceled) && !strings.Contains(strings.ToLower(err.Error()), "canceled") { - t.Fatalf("expected cancellation-related error, got %v", err) - } - if atomic.LoadInt32(&attempts) > 1 { - t.Fatalf("expected no retry after cancellation, got attempts=%d", attempts) - } -} - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { - return f(r) -} - -func newJSONHTTPResponse(status int, body string) *http.Response { - return &http.Response{ - StatusCode: status, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(body)), - } -} diff --git a/internal/framework/llm/secrets.go b/internal/framework/llm/secrets.go index 9322404..b7aee83 100644 --- a/internal/framework/llm/secrets.go +++ b/internal/framework/llm/secrets.go @@ -8,10 +8,11 @@ func ConfiguredSecrets(cfg *config.Config) []string { if cfg == nil { return nil } - effectiveValidation := cfg.EffectiveValidationLLMConfig() - return []string{ - cfg.PrimaryLLM.APIKey, - cfg.ValidationLLM.APIKey, - effectiveValidation.APIKey, + secrets := make([]string, 0, len(cfg.Backends)) + for _, backend := range cfg.Backends { + if backend.Credential != nil && backend.Credential.Value() != "" { + secrets = append(secrets, backend.Credential.Value()) + } } + return secrets } diff --git a/internal/framework/llm/secrets_test.go b/internal/framework/llm/secrets_test.go index c7be874..4fd626d 100644 --- a/internal/framework/llm/secrets_test.go +++ b/internal/framework/llm/secrets_test.go @@ -1,38 +1,31 @@ package llm import ( - "reflect" "testing" + ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" ) -func TestConfiguredSecretsNilConfig(t *testing.T) { - if got := ConfiguredSecrets(nil); got != nil { - t.Fatalf("expected nil secrets for nil config, got %v", got) - } -} - -func TestConfiguredSecretsWithValidationOverride(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = "primary-secret" - cfg.ValidationLLM.APIKey = "validation-secret" - - got := ConfiguredSecrets(&cfg) - want := []string{"primary-secret", "validation-secret", "validation-secret"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("unexpected secrets: got=%v want=%v", got, want) - } -} - -func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) { - cfg := config.Default() - cfg.PrimaryLLM.APIKey = "primary-secret" - cfg.ValidationLLM.APIKey = "" - - got := ConfiguredSecrets(&cfg) - want := []string{"primary-secret", "", "primary-secret"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want) +func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) { + if ConfiguredSecrets(nil) != nil { + t.Fatal("nil configuration has secrets") + } + t.Setenv("SELECTED_KEY", "selected-secret") + t.Setenv("OTHER_KEY", "unselected-secret") + loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "") + if err != nil { + t.Fatal(err) + } + got := ConfiguredSecrets(&loaded.Config) + if len(got) != 2 { + t.Fatalf("secrets count = %d", len(got)) + } + seen := map[string]bool{} + for _, value := range got { + seen[value] = true + } + if !seen["selected-secret"] || !seen["unselected-secret"] { + t.Fatal("missing resolved backend secret") } } diff --git a/internal/framework/structuredoutput/call.go b/internal/framework/structuredoutput/call.go index 824273f..981084c 100644 --- a/internal/framework/structuredoutput/call.go +++ b/internal/framework/structuredoutput/call.go @@ -3,6 +3,7 @@ package structuredoutput import ( "context" "encoding/json" + "time" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" ) @@ -39,13 +40,38 @@ type PromptMetadata struct { } type Response struct { - Content json.RawMessage `json:"content,omitempty"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - Prompt PromptMetadata `json:"prompt"` + Content json.RawMessage `json:"content,omitempty"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + Prompt PromptMetadata `json:"prompt"` + Prepared *PreparedSnapshot `json:"prepared,omitempty"` + FinalContent string `json:"final_content,omitempty"` +} + +// PreparedSnapshot is the exact rendered request snapshot. Its messages may +// contain transcript and glossary content and belong only in protected diagnostics. +type PreparedSnapshot struct { + Messages []Message `json:"messages"` + Prompt PromptMetadata `json:"prompt"` +} + +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +// CallRecord brackets one logical operation, including preparation. Content +// is present only when a model returned it; consumers must protect it. +type CallRecord struct { + Context DiagnosticContext `json:"context"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + Prepared *PreparedSnapshot `json:"prepared,omitempty"` + FinalContent string `json:"final_content,omitempty"` + Failure FailureKind `json:"failure,omitempty"` } type Client interface { diff --git a/internal/framework/structuredoutput/failure.go b/internal/framework/structuredoutput/failure.go new file mode 100644 index 0000000..78b7357 --- /dev/null +++ b/internal/framework/structuredoutput/failure.go @@ -0,0 +1,27 @@ +package structuredoutput + +import "errors" + +// FailureKind separates invalid generated content from operational failures. +type FailureKind string + +const ( + FailureMalformedOutput FailureKind = "malformed_output" + FailureRuntime FailureKind = "runtime" +) + +// Failure exposes a safe message while retaining the typed cause for callers. +// The cause must not be serialized or formatted directly in user artifacts. +type Failure struct { + Kind FailureKind + Message string + Cause error +} + +func (f *Failure) Error() string { return f.Message } +func (f *Failure) Unwrap() error { return f.Cause } + +func IsMalformedError(err error) bool { + var failure *Failure + return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput +} diff --git a/internal/framework/structuredoutput/failure_test.go b/internal/framework/structuredoutput/failure_test.go new file mode 100644 index 0000000..1287f02 --- /dev/null +++ b/internal/framework/structuredoutput/failure_test.go @@ -0,0 +1,18 @@ +package structuredoutput + +import ( + "errors" + "testing" +) + +func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) { + cause := errors.New("provider token secret-value") + malformed := &Failure{Kind: FailureMalformedOutput, Message: "malformed structured output", Cause: cause} + if !IsMalformedError(malformed) || !errors.Is(malformed, cause) || malformed.Error() != "malformed structured output" { + t.Fatalf("malformed classification: %v", malformed) + } + runtime := &Failure{Kind: FailureRuntime, Message: "structured model call failed", Cause: cause} + if IsMalformedError(runtime) || IsMalformedError(cause) || IsMalformedError(nil) || !errors.Is(runtime, cause) || runtime.Error() != "structured model call failed" { + t.Fatalf("runtime classification: %v", runtime) + } +} diff --git a/internal/framework/structuredoutput/malformed.go b/internal/framework/structuredoutput/malformed.go deleted file mode 100644 index 01fe815..0000000 --- a/internal/framework/structuredoutput/malformed.go +++ /dev/null @@ -1,28 +0,0 @@ -package structuredoutput - -import "strings" - -var malformedMarkers = []string{ - "malformed structured output", - "decode structured output:", - "decode provider response envelope:", - "provider response missing choices", - "provider response missing assistant message content", - "provider response assistant message content is empty", - "provider response assistant message content is not valid JSON", -} - -// IsMalformedError reports whether err matches provider malformed -// structured-output failure markers that should be downgraded. -func IsMalformedError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - for _, marker := range malformedMarkers { - if strings.Contains(msg, marker) { - return true - } - } - return false -} diff --git a/internal/framework/structuredoutput/malformed_test.go b/internal/framework/structuredoutput/malformed_test.go deleted file mode 100644 index fd5c08a..0000000 --- a/internal/framework/structuredoutput/malformed_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package structuredoutput - -import ( - "errors" - "testing" -) - -func TestIsMalformedError(t *testing.T) { - cases := []struct { - name string - err error - want bool - }{ - {name: "nil", err: nil, want: false}, - {name: "generic", err: errors.New("network timeout"), want: false}, - {name: "malformed", err: errors.New("malformed structured output"), want: true}, - {name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true}, - {name: "missing choices", err: errors.New("provider response missing choices"), want: true}, - {name: "missing content", err: errors.New("provider response missing assistant message content"), want: true}, - {name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true}, - {name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := IsMalformedError(tc.err) - if got != tc.want { - t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want) - } - }) - } -}