From 3d3f16db4aac65c6e5bf0106aaeab66d9cc1863c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 25 Aug 2026 20:04:56 +0000 Subject: [PATCH] Enable structural output repair by default --- assets/dnd/combat-turns/prompts/prompt.yaml | 2 +- assets/dnd/enemy-events/prompts/prompt.yaml | 2 +- .../dnd/item-occurrences/prompts/prompt.yaml | 2 +- .../item-registry/extract/prompts/prompt.yaml | 2 +- .../normalize/prompts/prompt.yaml | 2 +- .../location-occurrences/prompts/prompt.yaml | 2 +- .../extract/prompts/prompt.yaml | 2 +- .../normalize/prompts/prompt.yaml | 2 +- .../dnd/npc-occurrences/prompts/prompt.yaml | 2 +- .../npc-registry/extract/prompts/prompt.yaml | 2 +- .../normalize/prompts/prompt.yaml | 2 +- .../scene-descriptions/prompts/prompt.yaml | 2 +- assets/dnd/scenes/prompts/prompt.yaml | 2 +- assets/dnd/spells/prompts/prompt.yaml | 2 +- .../deduplication/prompts/prompt.yaml | 2 +- docs/config.md | 4 +- docs/integrations/pkg-promptkit.md | 8 +- docs/internal/llm.md | 31 +++-- docs/internal/pipeline.md | 3 + docs/operations.md | 18 ++- docs/policy/architecture.md | 4 + docs/roadmap/implementation.md | 2 +- internal/cli/production_contract_test.go | 58 ++++++-- .../framework/llm/promptkit_client_test.go | 131 +++++++++++++++++- .../semanticreconcile/assets_test.go | 4 +- 25 files changed, 241 insertions(+), 52 deletions(-) diff --git a/assets/dnd/combat-turns/prompts/prompt.yaml b/assets/dnd/combat-turns/prompts/prompt.yaml index 7aeec24e..5d906f0a 100644 --- a/assets/dnd/combat-turns/prompts/prompt.yaml +++ b/assets/dnd/combat-turns/prompts/prompt.yaml @@ -42,4 +42,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_combat_turns_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/enemy-events/prompts/prompt.yaml b/assets/dnd/enemy-events/prompts/prompt.yaml index 7f73d248..43465ba8 100644 --- a/assets/dnd/enemy-events/prompts/prompt.yaml +++ b/assets/dnd/enemy-events/prompts/prompt.yaml @@ -50,4 +50,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_enemy_events_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/item-occurrences/prompts/prompt.yaml b/assets/dnd/item-occurrences/prompts/prompt.yaml index a8fef9b0..7478db5c 100644 --- a/assets/dnd/item-occurrences/prompts/prompt.yaml +++ b/assets/dnd/item-occurrences/prompts/prompt.yaml @@ -42,4 +42,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_item_occurrences_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/item-registry/extract/prompts/prompt.yaml b/assets/dnd/item-registry/extract/prompts/prompt.yaml index c380ba46..67f40598 100644 --- a/assets/dnd/item-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/item-registry/extract/prompts/prompt.yaml @@ -37,4 +37,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_item_registry_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/item-registry/normalize/prompts/prompt.yaml b/assets/dnd/item-registry/normalize/prompts/prompt.yaml index c1d8ecee..02a5855f 100644 --- a/assets/dnd/item-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/item-registry/normalize/prompts/prompt.yaml @@ -27,4 +27,4 @@ output: format: json validation_mode: json_schema schema_path: semantic_reconciliation_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/location-occurrences/prompts/prompt.yaml b/assets/dnd/location-occurrences/prompts/prompt.yaml index fec90d41..92f28b2a 100644 --- a/assets/dnd/location-occurrences/prompts/prompt.yaml +++ b/assets/dnd/location-occurrences/prompts/prompt.yaml @@ -42,4 +42,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_location_occurrences_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/location-registry/extract/prompts/prompt.yaml b/assets/dnd/location-registry/extract/prompts/prompt.yaml index 485c4f72..7d4d3dd3 100644 --- a/assets/dnd/location-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/location-registry/extract/prompts/prompt.yaml @@ -37,4 +37,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_location_registry_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/location-registry/normalize/prompts/prompt.yaml b/assets/dnd/location-registry/normalize/prompts/prompt.yaml index b23d1b72..12ef271b 100644 --- a/assets/dnd/location-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/location-registry/normalize/prompts/prompt.yaml @@ -27,4 +27,4 @@ output: format: json validation_mode: json_schema schema_path: semantic_reconciliation_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/npc-occurrences/prompts/prompt.yaml b/assets/dnd/npc-occurrences/prompts/prompt.yaml index 64549110..f9a85d53 100644 --- a/assets/dnd/npc-occurrences/prompts/prompt.yaml +++ b/assets/dnd/npc-occurrences/prompts/prompt.yaml @@ -42,4 +42,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_npc_occurrences_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/npc-registry/extract/prompts/prompt.yaml b/assets/dnd/npc-registry/extract/prompts/prompt.yaml index c119b5d7..1a9f071c 100644 --- a/assets/dnd/npc-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/npc-registry/extract/prompts/prompt.yaml @@ -37,4 +37,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_npc_registry_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/npc-registry/normalize/prompts/prompt.yaml b/assets/dnd/npc-registry/normalize/prompts/prompt.yaml index b71c57e8..0be55874 100644 --- a/assets/dnd/npc-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/npc-registry/normalize/prompts/prompt.yaml @@ -27,4 +27,4 @@ output: format: json validation_mode: json_schema schema_path: semantic_reconciliation_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/scene-descriptions/prompts/prompt.yaml b/assets/dnd/scene-descriptions/prompts/prompt.yaml index fc12786e..61b0f222 100644 --- a/assets/dnd/scene-descriptions/prompts/prompt.yaml +++ b/assets/dnd/scene-descriptions/prompts/prompt.yaml @@ -35,4 +35,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_scene_descriptions_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/scenes/prompts/prompt.yaml b/assets/dnd/scenes/prompts/prompt.yaml index 2b06d84d..51f638ef 100644 --- a/assets/dnd/scenes/prompts/prompt.yaml +++ b/assets/dnd/scenes/prompts/prompt.yaml @@ -31,4 +31,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_scenes_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/dnd/spells/prompts/prompt.yaml b/assets/dnd/spells/prompts/prompt.yaml index 4e5b152c..529ba85c 100644 --- a/assets/dnd/spells/prompts/prompt.yaml +++ b/assets/dnd/spells/prompts/prompt.yaml @@ -47,4 +47,4 @@ output: format: json validation_mode: json_schema schema_path: dnd_spells_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/assets/generic/normalize/deduplication/prompts/prompt.yaml b/assets/generic/normalize/deduplication/prompts/prompt.yaml index 8f07b2ad..013fe707 100644 --- a/assets/generic/normalize/deduplication/prompts/prompt.yaml +++ b/assets/generic/normalize/deduplication/prompts/prompt.yaml @@ -24,4 +24,4 @@ output: format: json validation_mode: json_schema schema_path: semantic_reconciliation_llm.v1.json - repair_attempts: 0 + repair_attempts: 1 diff --git a/docs/config.md b/docs/config.md index 166b98a9..d912ea2b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -222,7 +222,7 @@ pipelines: | Field | Type | Default | Rules | | --- | --- | --- | --- | | **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. | -| **structured_output_repair_attempts** | integer | prompt-owned | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. | +| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. | | **input** | module binding | none | Required. | | **chunk** | module binding | **generic** | Optional. | | **output** | module binding | **json** | Optional. | @@ -288,7 +288,7 @@ extract: | --- | --- | --- | --- | | **module** | string | none | Required for an object binding. Must be a registered compatible key. | | **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. | -| **structured_output_repair_attempts** | integer | pipeline or prompt-owned | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. | +| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. | | **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. | | **options** | object | none | Must satisfy the selected module. | | **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. | diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index 21d1a693..2091b024 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -115,7 +115,7 @@ the conventional local backend. PromptKit API or format changes outside this boundary are not implicitly supported. Updating the pinned version requires reviewing the adapter and -profile/configuration contracts against the upstream documentation. - -This dependency update does not change any production prompt's declared -`repair_attempts` budget. +profile/configuration contracts against the upstream documentation. Maintained +production prompts use PromptKit's bounded structural-repair contract; their +current declaration is one additional repair attempt. Notarius retains the +transport-neutral boundary and does not expose PromptKit types to modules. diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 3a25aaaa..1a37a392 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -210,14 +210,29 @@ caller context takes precedence. The adapter does not retry capacity failures; the pipeline's existing binding attempt policy sees the operational error and decides whether to rerun the complete operation. -Prompt-declared repair is executed within PromptKit’s structured-output flow. -The current production D&D prompt manifests set repair attempts to zero. That -setting does not replace pipeline retry behavior: a binding’s configured retry -count reruns its stage attempt after an error or rejection, and an exhausted -rejection is a recorded output rather than a provider error. The pipeline owns -attempt lifecycle, validation chains, and retry diagnostics; see -[Pipeline Internals](pipeline.md#validation-retries-and-output) and the -[binding reference](../config.md#module-bindings-and-validators). +PromptKit executes structural repair within its structured-output flow. The +maintained production prompt manifests declare one additional repair attempt. +When a resolved binding supplies a repair value, the adapter inspects the +prompt, copies its complete output contract, changes only the repair limit, and +passes that complete replacement contract to PromptKit. This preserves the +prompt's output format, validation mode, schema, and provider structured-output +settings. + +A successful repair is an ordinary successful completion, not a warning. The +adapter reports PromptKit's actual repair count and its cumulative usage +directly, without adding the initial and corrective counts again. Debug prompt +material records the configured complete contract; debug response material +records the repaired response and actual validation result. If the repair +budget is exhausted, the adapter retains the final raw bytes and debug material +and reports `ErrInvalidStructuredOutput`. Generation failures during an initial +or corrective call remain provider-neutral operational errors with the same +redaction boundary. + +Structural repair does not replace pipeline retry behavior: a binding's +configured retry count reruns its complete stage attempt after an error or +rejection. The pipeline owns attempt lifecycle, validation chains, and retry +diagnostics; see [Pipeline Internals](pipeline.md#validation-retries-and-output) +and the [binding reference](../config.md#module-bindings-and-validators). ## Timeout Ownership diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 758e6bb7..15ef8441 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -43,6 +43,9 @@ leaves the prompt-owned policy intact. An explicit repair value on a deterministic binding is rejected. Resolved bindings own copied repair values, and these effective values are part of the digest, so execution and checkpoint consumers do not repeat profile inheritance or configuration resolution. +Each LLM request receives its own copy of that resolved value. PromptKit spends +it only for structural correction inside one completion; the runner's binding +retry policy remains the separate outer budget for complete stage attempts. Configuration resolution supplies the selected profile and catalog; see [Configuration Internals](configuration.md). diff --git a/docs/operations.md b/docs/operations.md index 3effa0f3..0734a3a4 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -281,12 +281,23 @@ selected binding, then the pipeline, then the prompt declaration; see [module bindings](config.md#module-bindings-and-validators). This is distinct from Notarius binding **retries**, which rerun the complete module operation and validation chain and do not consume or replenish the structural-repair -limit. +limit. The maintained production prompts declare one repair attempt, paid only +after a structural failure. One structured completion with repair budget **R** +makes at most **R + 1** serial provider calls. If one stage attempt performs +**C** structured completions, a binding with **retries: N** has a maximum of +**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own +corresponding invocation counts and budgets. This is an upper bound, not a +promise that every call reaches a provider. Timeouts are layered. Caller cancellation is the outer authority. A positive effective generation timeout adds an inner request deadline, while zero disables only that generation deadline. The HTTP client timeout remains a transport-wide cap. Notarius does not add another timeout around PromptKit. +Repairs are serial within the same caller context, so their worst-case latency +and cost follow the provider-call bound above; provision run deadlines and +provider budgets accordingly. Credentials remain optional unless the selected +PromptKit profile requires one, in which case preparation fails before a +provider call when its configured credential is unavailable. The pinned upstream boundary and profile-format links are in [PromptKit Integration](integrations/pkg-promptkit.md). @@ -303,6 +314,11 @@ positive value makes the effective active local-generation bound the smaller of **total_llm** and that local limit, so a local limit of four permits no more than four active local generations. +The Notarius scheduler admits one logical structured completion and holds that +permit while PromptKit performs its serial corrective calls. PromptKit applies +its selected-backend admission to each provider call; Notarius does not +reacquire a permit or add another scheduler for a repair. + For a positive local limit, PromptKit owns its default waiting capacity and admission behavior. When a PromptKit backend has admitted all active and queued work, a new call fails as capacity exhaustion before generation. The adapter diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 296709f2..c8a389f3 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -195,6 +195,10 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema, and interpretation of structured output. Provider adapters do not own source- or domain-specific prompt logic. +PromptKit owns bounded structural correction within one structured completion. +Notarius owns outer stage attempts, semantic validation, and acceptance policy; +the two budgets must remain separate. + When a model selects an application entity, callers must supply a contextual selection and deterministically attach the opaque application identity whenever the selection resolves exactly. Models do not receive or reproduce opaque diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 54f7728b..f3683f39 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -676,7 +676,7 @@ git diff --check - Current configuration, pipeline, and operations documentation matches the implemented behavior. -## Stage 10: Enable The Default, Finish Documentation, And Verify The Feature +## Stage 10: Enable The Default, Finish Documentation, And Verify The Feature ✅ ### Goal diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index 8ae2cf22..0f9942ae 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -19,6 +19,8 @@ import ( "testing/fstest" "time" + "gopkg.in/yaml.v3" + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" @@ -331,27 +333,53 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) { if err != nil { t.Fatalf("production prompt engine: %v", err) } - inputs := map[string]promptkit.ArtifactRef{ - "candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Alias","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), - "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`), + promptFS, err := components.assets.PromptFS() + if err != nil { + t.Fatalf("production prompt assets: %v", err) } - for _, prompt := range []struct { - id string - version string - }{ - {id: npcnormalize.PromptID, version: npcnormalize.PromptVersion}, - {id: itemregistrynormalize.PromptID, version: itemregistrynormalize.PromptVersion}, - {id: locationnormalize.PromptID, version: locationnormalize.PromptVersion}, - } { + type manifest struct { + ID string `yaml:"id"` + Version string `yaml:"version"` + Inputs []struct { + Name string `yaml:"name"` + } `yaml:"inputs"` + } + preparedPrompts := 0 + if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Base(path) != "prompt.yaml" { + return nil + } + data, err := fs.ReadFile(promptFS, path) + if err != nil { + return err + } + var prompt manifest + if err := yaml.Unmarshal(data, &prompt); err != nil { + return err + } + inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs)) + for _, input := range prompt.Inputs { + inputs[input.Name] = promptkit.Inline(`{}`) + } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: prompt.id, PromptVersion: prompt.version, ProfileID: "assembled-prompt-test", Inputs: inputs, + PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs, }) if err != nil { - t.Fatalf("prepare production prompt %q: %v", prompt.id, err) + return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err) } - if prepared.OutputContract.SchemaPath != filepath.Base(semanticreconcile.SchemaAssetPath) { - t.Fatalf("prompt %q schema = %q, want generic reconciliation schema", prompt.id, prepared.OutputContract.SchemaPath) + if prepared.OutputContract.RepairAttempts != 1 { + return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts) } + preparedPrompts++ + return nil + }); err != nil { + t.Fatal(err) + } + if preparedPrompts == 0 { + t.Fatal("prepared no production prompts") } } diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 1a6d9a3a..99092e19 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "io/fs" "net/http" @@ -922,12 +923,15 @@ func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) { } func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { - client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`}) + attempts := 1 + fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{{Content: `{"bad":true}`}, {Content: `{"bad":true}`}}} + client := newTestPromptKitClient(t, fake) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - PromptID: "adapter.test", - SessionID: "session-123", + PromptID: "adapter.test", + SessionID: "session-123", + StructuredOutputRepairAttempts: &attempts, Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, @@ -935,6 +939,9 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") { t.Fatalf("CompleteStructured() error = %v, want validation failure", err) } + if got := atomic.LoadInt32(&fake.calls); got != 2 { + t.Fatalf("provider calls = %d, want exhausted repair budget", got) + } if got := string(resp.Content); got != `{"bad":true}` { t.Fatalf("response content = %q, want raw failed output", got) } @@ -946,6 +953,45 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { } } +func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *testing.T) { + attempts := 1 + fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{ + {Content: `{"bad":true}`, Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8}}, + {Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}}, + }} + client := newTestPromptKitClient(t, fake) + + var out struct { + OK bool `json:"ok"` + } + response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + StructuredOutputRepairAttempts: &attempts, + SessionID: "repair-test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err != nil || !out.OK { + t.Fatalf("CompleteStructured() = (%#v, %v), want repaired success", response, err) + } + if got := atomic.LoadInt32(&fake.calls); got != 2 { + t.Fatalf("provider calls = %d, want initial generation and one repair", got) + } + if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 { + t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response) + } + if response.Debug == nil || response.Debug.Prompt == nil || response.Debug.Response == nil { + t.Fatalf("debug = %#v, want prompt and response details", response.Debug) + } + if response.Debug.Prompt.OutputContract["repair_attempts"] != float64(1) || + response.Debug.Response.Validation["repair_attempts"] != float64(1) || + response.Debug.Response.Content != `{"ok":true}` || + response.Debug.Response.Usage.TotalTokens != 26 { + t.Fatalf("debug repair result = %#v, want configured contract and repaired response", response.Debug) + } +} + func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) @@ -1221,6 +1267,66 @@ func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) { wg.Wait() } +func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) { + attempts := 1 + fake := &fakePromptKitLLM{ + responses: []promptkit.GenerateResponse{ + {Content: `{"bad":true}`}, + {Content: `{"ok":true}`}, + {Content: `{"ok":true}`}, + }, + block: make(chan struct{}), + } + scheduler, err := NewScheduler(1) + if err != nil { + t.Fatalf("NewScheduler() error = %v", err) + } + scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler) + + firstDone := make(chan error, 1) + go func() { + var out map[string]any + _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "repair-session", + StructuredOutputRepairAttempts: &attempts, + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + firstDone <- callErr + }() + waitForAtomicAtLeast(t, &fake.calls, 1) + + secondDone := make(chan error, 1) + go func() { + var out map[string]any + _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "queued-session", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + secondDone <- callErr + }() + close(fake.block) + if err := <-firstDone; err != nil { + t.Fatalf("repaired completion error = %v", err) + } + if err := <-secondDone; err != nil { + t.Fatalf("queued completion error = %v", err) + } + + requests := fake.requestsSnapshot() + if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" { + t.Fatalf("generation order = %#v, want repair before queued completion", requests) + } + if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 { + t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got) + } +} + func TestPromptKitClientValidatesRequest(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out map[string]any @@ -1375,11 +1481,13 @@ func (f *switchingPromptFS) resumePromptRead() { type fakePromptKitLLM struct { content string + responses []promptkit.GenerateResponse allowEmpty bool err error block chan struct{} mu sync.Mutex last promptkit.GenerateRequest + requests []promptkit.GenerateRequest calls int32 inFlight int32 maxInFlight int32 @@ -1394,8 +1502,9 @@ func (*credentialBearingProviderError) Error() string { func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { f.mu.Lock() f.last = req + f.requests = append(f.requests, req) f.mu.Unlock() - atomic.AddInt32(&f.calls, 1) + call := atomic.AddInt32(&f.calls, 1) current := atomic.AddInt32(&f.inFlight, 1) for { seen := atomic.LoadInt32(&f.maxInFlight) @@ -1414,6 +1523,14 @@ func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateR if f.err != nil { return nil, f.err } + if len(f.responses) > 0 { + index := int(call - 1) + if index >= len(f.responses) { + return nil, fmt.Errorf("unexpected provider call %d", call) + } + response := f.responses[index] + return &response, nil + } content := f.content if content == "" && !f.allowEmpty { content = `{"ok":true}` @@ -1444,3 +1561,9 @@ func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest { defer f.mu.Unlock() return f.last } + +func (f *fakePromptKitLLM) requestsSnapshot() []promptkit.GenerateRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]promptkit.GenerateRequest(nil), f.requests...) +} diff --git a/internal/framework/semanticreconcile/assets_test.go b/internal/framework/semanticreconcile/assets_test.go index 5b7b3680..c703c6ef 100644 --- a/internal/framework/semanticreconcile/assets_test.go +++ b/internal/framework/semanticreconcile/assets_test.go @@ -45,8 +45,8 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) { if prepared.SelectedProfileID != "semantic-reconciliation-test" { t.Fatalf("selected profile = %q, want explicit test profile", prepared.SelectedProfileID) } - if contract := prepared.OutputContract; contract.SchemaPath != "semantic_reconciliation_llm.v1.json" || contract.RepairAttempts != 0 { - t.Fatalf("output contract = %#v, want generic schema without repair", contract) + if contract := prepared.OutputContract; contract.SchemaPath != "semantic_reconciliation_llm.v1.json" || contract.RepairAttempts != 1 { + t.Fatalf("output contract = %#v, want generic schema with one repair", contract) } if len(prepared.Messages) != 5 || prepared.Messages[0].Role != "system" { t.Fatalf("prepared messages = %#v, want five ordered messages beginning with system", prepared.Messages)