Compare commits

45 Commits

Author SHA1 Message Date
e92bcfa74c Prepare the v0.4.0 release 2026-08-27 13:30:24 +00:00
e6b2ae88d2 Strengthen validation retries and checkpoint safety 2026-08-27 11:50:37 +00:00
b2e83bd6e7 Document feedback-aware validation retries 2026-08-27 01:35:23 +00:00
7f01c3e79e Add durable validation provenance 2026-08-27 01:27:25 +00:00
7b5f4ebd42 Apply validation retries to merge and normalize 2026-08-27 01:13:32 +00:00
4bb4582695 Integrate extraction validation retries 2026-08-27 01:06:17 +00:00
7d3434e5f6 Expose typed validation reports 2026-08-27 00:37:20 +00:00
29872b2e28 Integrate chunk validation retries 2026-08-27 00:35:37 +00:00
b487d93186 Add producer attempt state machine 2026-08-27 00:27:27 +00:00
8dd7a4324d Execute complete validator chains 2026-08-27 00:18:16 +00:00
04ba87e174 Enable correction-aware semantic reconciliation 2026-08-27 00:08:05 +00:00
a26d6ed042 Enable correction-aware downstream D&D extractors 2026-08-26 23:59:05 +00:00
759d32403f Enable correction-aware foundational D&D producers 2026-08-26 23:55:23 +00:00
1a7b20c766 Adapt corrections through PromptKit 2026-08-26 23:50:24 +00:00
85a5b52be7 Declare producer correction capability 2026-08-26 23:42:11 +00:00
9d0faabf61 Add validation policy configuration 2026-08-26 23:34:06 +00:00
1c3da3e869 Add feedback-aware correction contracts 2026-08-26 23:24:20 +00:00
9abd93502f Upgrade PromptKit to v0.9.0 2026-08-26 23:12:16 +00:00
c8a29a5fa2 Plan feedback-aware validation retries 2026-08-26 18:51:36 +00:00
916d9210fd Preserve structured repair settings across pipeline boundaries 2026-08-25 23:46:52 +00:00
3d3f16db4a Enable structural output repair by default 2026-08-25 20:04:56 +00:00
63c397d86a Resolve structured output repair configuration 2026-08-25 19:55:39 +00:00
9a92212632 Add structured output repair configuration 2026-08-25 19:47:42 +00:00
0ef8931697 Forward repair policy from D&D modules 2026-08-25 19:40:44 +00:00
e00cc45c6b Propagate structured repair requests 2026-08-25 19:39:04 +00:00
ab9b743df6 Add structured output repair support 2026-08-25 19:35:27 +00:00
3bd3c7ebf7 Classify PromptKit generation errors safely 2026-08-25 19:34:27 +00:00
75a3f51cee Support inherited PromptKit profiles 2026-08-25 19:31:00 +00:00
2be999ebd3 Verify PromptKit compatibility boundaries 2026-08-25 19:27:34 +00:00
32fe7c5b98 Upgrade PromptKit to v0.8.0 2026-08-25 19:25:57 +00:00
55247c47ab Plan the PromptKit 0.8 upgrade 2026-08-25 19:11:30 +00:00
5e5c69bf9d Document future validation and retry work 2026-08-25 14:29:00 +00:00
b4a81f8b09 Validate release versions before tagging 2026-08-25 02:21:46 +00:00
adfed22e1a Complete source release system verification 2026-08-25 02:05:17 +00:00
75b1e2f68b Document source release deployment 2026-08-25 02:03:11 +00:00
4473363d9f Document source release procedure 2026-08-25 02:01:52 +00:00
6eb45e0003 Add release tag validation workflow 2026-08-25 01:58:29 +00:00
0585ad76dc Add source release candidate checks 2026-08-25 01:57:10 +00:00
56145c3b7e Add build version reporting 2026-08-25 01:55:09 +00:00
a478fd86c5 Plan source-only releases 2026-08-25 01:48:16 +00:00
916532100d Add D&D consumer documentation 2026-08-09 20:57:28 +00:00
bef8ca263b Tighten evidence context source excerpts 2026-08-09 20:04:46 +00:00
f6d037b613 Document evidence context source unit excerpts 2026-08-09 19:47:27 +00:00
449b506804 Update evidence context output coverage 2026-08-09 19:45:21 +00:00
071a78ae22 Replace evidence context with source unit excerpts 2026-08-09 19:42:33 +00:00
216 changed files with 8623 additions and 1895 deletions

33
.woodpecker/release.yml Normal file
View File

@@ -0,0 +1,33 @@
when:
- event: tag
steps:
- name: validate-release
image: golang:1.25.5
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
release_note="docs/releases/$version.md"
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
printf '%s\n' "invalid release tag: $version" >&2
exit 1
fi
if [ ! -s "$release_note" ]; then
printf '%s\n' "missing release note: $release_note" >&2
exit 1
fi
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
printf '%s\n' "release note heading does not match $version" >&2
exit 1
fi
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
printf '%s\n' "release note is missing heading: $heading" >&2
exit 1
fi
done
./scripts/check-release-source.sh "$version"

View File

@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
[its synthetic transcript](examples/dnd-complete-transcript.json). It [its synthetic transcript](examples/dnd-complete-transcript.json). It
demonstrates all implemented D&D lanes and the supporting campaign references. demonstrates all implemented D&D lanes and the supporting campaign references.
## Install A Source Release
Install a pinned source release with Go:
~~~
GOWORK=off go install \
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
~~~
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
installed command's diagnostic version is described in the [CLI
reference](docs/cli.md); maintainers preparing a release should follow [Source
Releases](docs/release.md).
## Documentation ## Documentation
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits. - [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
artifact formats. artifact formats.
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius - [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
from an orchestrator and consume a published result. from an orchestrator and consume a published result.
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
D&D pipeline as a subprocess and discover its structured artifacts.
- [Internal overview](docs/internal/overview.md) — implemented component map - [Internal overview](docs/internal/overview.md) — implemented component map
for maintainers. for maintainers.
- [Developer guide](docs/development.md) — contributor orientation and - [Developer guide](docs/development.md) — contributor orientation and

View File

@@ -42,4 +42,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_combat_turns_llm.v1.json schema_path: dnd_combat_turns_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -50,4 +50,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_enemy_events_llm.v1.json schema_path: dnd_enemy_events_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -42,4 +42,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_item_occurrences_llm.v1.json schema_path: dnd_item_occurrences_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_item_registry_llm.v1.json schema_path: dnd_item_registry_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -27,4 +27,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: semantic_reconciliation_llm.v1.json schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -42,4 +42,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_location_occurrences_llm.v1.json schema_path: dnd_location_occurrences_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_location_registry_llm.v1.json schema_path: dnd_location_registry_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -27,4 +27,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: semantic_reconciliation_llm.v1.json schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -42,4 +42,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_npc_occurrences_llm.v1.json schema_path: dnd_npc_occurrences_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_npc_registry_llm.v1.json schema_path: dnd_npc_registry_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -27,4 +27,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: semantic_reconciliation_llm.v1.json schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -35,4 +35,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_scene_descriptions_llm.v1.json schema_path: dnd_scene_descriptions_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -31,4 +31,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_scenes_llm.v1.json schema_path: dnd_scenes_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -47,4 +47,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: dnd_spells_llm.v1.json schema_path: dnd_spells_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -24,4 +24,4 @@ output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: semantic_reconciliation_llm.v1.json schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 0 repair_attempts: 1

View File

@@ -83,9 +83,7 @@ entity identity, checkpoint contracts, or cross-request correlation. Changes
to shared protocol and policy assets must participate in the normal prompt, to shared protocol and policy assets must participate in the normal prompt,
schema, and checkpoint fingerprint mechanisms. schema, and checkpoint fingerprint mechanisms.
Acceptance of this decision does not imply that the shared mechanism or its The shared mechanism and its initial D&D registry consumers are now
consumer migrations are implemented. The implemented. Current behavior is documented in
[feature roadmap](../roadmap/semantic-reconciliation.md) owns target behavior [Module Internals](../internal/modules.md#semantic-reconciliation) and
and status, and the [D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
[implementation plan](../roadmap/implementation.md) owns delivery sequence
until the work is complete.

View File

@@ -0,0 +1,59 @@
# ADR-0014: Use feedback-aware validation retries
**Status:** Accepted
**Date:** 2026-08-26
## Context
Validation can identify a candidate defect after a producer has returned an
otherwise well-formed result. Retrying without the validator's deterministic,
bounded feedback wastes the useful diagnosis, while treating validator
execution failures as defects would ask a producer to repair conditions it
cannot control. The mechanism must preserve typed producer ownership,
checkpoint safety, and the repository's sensitive-data boundaries.
## Decision
The implementation will keep three independent budgets: the producer binding's
outer `retries` budget, PromptKit's structured-output repair budget, and each
validator's execution-retry budget. Validators will run sequentially in their
configured order and aggregate both rejections and execution failures before a
candidate disposition is selected.
A correction-capable producer will provide the exact single LLM response that
controlled its candidate using the `single_response_v1` protocol. A correction
attempt will reconstruct the ordinary request and append exactly two fresh
messages: that latest response as `assistant`, followed by one deterministic
aggregate correction request as `user`. Earlier turns will not accumulate.
Validator failures will not recurse into correction. Pipeline policy owns
terminal disposition, with field-by-field producer overrides over pipeline
defaults: structural failure and semantic rejection default to `fail_run`, and
validator execution failure defaults to `warn_continue`. Validators can report
facts and bounded corrective guidance, but never decide disposition.
Rejected and structurally invalid candidates will not advance. A candidate
allowed through after a validator execution failure will retain explicit
incomplete-validation provenance and will not be checkpointed. Exact response
and correction text remain attempt-local: they are excluded from ordinary
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
summaries.
## Alternatives considered
- Retry every producer after any validation outcome. This conflates producer
defects with validator operational failures and wastes retry budget.
- Let validators decide whether to continue. This would distribute pipeline
disposition policy across validators and undermine consistent defaults.
- Reuse the full prior conversation. Accumulated turns introduce unbounded
prompt growth and make correction behavior depend on incidental history.
- Persist raw responses to simplify diagnosis. Raw model output and correction
guidance may be sensitive and do not belong in durable pipeline records.
## Consequences
The framework gains transport-neutral correction and candidate contracts,
producer capability checks, policy resolution, aggregated validation outcomes,
and conservative checkpoint handling. Prompt construction remains inside the
LLM adapter, while modules remain responsible for accurately exposing the
single response that directly controlled a candidate.

View File

@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
~~~ ~~~
notarius help notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags] notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--json] notarius pipelines list [--config path/to/config.yml] [--json]
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
Running Notarius without arguments, or with **help**, **--help**, or **-h**, Running Notarius without arguments, or with **help**, **--help**, or **-h**,
writes the command summary to standard output and exits with status 0. writes the command summary to standard output and exits with status 0.
`notarius --version` is valid only as the sole root argument. It writes exactly
`notarius <version>` followed by a newline to standard output and exits with
status 0. A tagged `go install` build can report its main-module stable tag,
and controlled builds can inject a stable tag at link time through
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
unversioned checkout reports `development`. Invalid injected version content is
a runtime error with exit status 1, while extra `--version` arguments are a
syntax error with exit status 2. This diagnostic does not replace the
[run-result](integrations/run-result.md) or artifact contracts for downstream
compatibility decisions.
## run ## run
~~~ ~~~

View File

@@ -132,7 +132,11 @@ model: example-model
Keep credentials out of the local-backend object. A PromptKit profile may name Keep credentials out of the local-backend object. A PromptKit profile may name
its credential environment variable through `api_key_env`; set that variable its credential environment variable through `api_key_env`; set that variable
only in the run environment. PromptKit owns the only in the run environment. PromptKit owns the
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md). [pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
including `base_profile` inheritance. Notarius passes profiles through without
merging them. Filesystem profiles cannot express PromptKit's in-memory
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
provider without authorization.
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
supported package API, and [Operations](operations.md#operational-limits) supported package API, and [Operations](operations.md#operational-limits)
describes the effective concurrency layers. describes the effective concurrency layers.
@@ -218,6 +222,8 @@ pipelines:
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. | | **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
| **input** | module binding | none | Required. | | **input** | module binding | none | Required. |
| **chunk** | module binding | **generic** | Optional. | | **chunk** | module binding | **generic** | Optional. |
| **output** | module binding | **json** | Optional. | | **output** | module binding | **json** | Optional. |
@@ -237,6 +243,42 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
then the pipeline's **llm_profile**, and finally the PromptKit default. then the pipeline's **llm_profile**, and finally the PromptKit default.
Deterministic bindings do not receive these defaults or run overrides. Deterministic bindings do not receive these defaults or run overrides.
Structural output repair is resolved after module, validator, and `--only` lane
selection. An object's **structured_output_repair_attempts** value takes
precedence over the pipeline value; otherwise, an LLM-backed binding or
validator inherits the pipeline value. If both are omitted, PromptKit uses the
prompt's declared repair policy. The value must be an integer from 0 through 3;
explicit `null` and non-integer values are invalid. An explicit value on a
deterministic binding or validator is invalid, while a pipeline value simply
does not apply to deterministic selections.
`validation_policy` controls terminal disposition for one complete producer
attempt and validator chain. It may appear on a pipeline or a **chunk**,
**extract**, **merge**, or **normalize** module binding; input, output, and
validator bindings reject it. Every field is optional and resolves in binding,
pipeline, then application-default order:
| Field | Values | Default |
| --- | --- | --- |
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
The policy object and its fields must be non-null, and unknown fields are
rejected. A deterministic producer may not explicitly set
**producer_structural_failure** on its binding, although a pipeline-level
default remains valid for pipelines that include LLM-backed producers.
After the producer binding's retry budget is exhausted, an invalid structured
response uses **producer_structural_failure**. One or more semantic validator
rejections use **semantic_rejection**; rejection takes precedence over an
exhausted validator failure or skip. With no rejection, an exhausted validator
failure or skip uses **validator_failure**. `reject_output` records the
terminal rejection without advancing that candidate. `warn_continue` is valid
only for validator execution failure: it advances a structurally valid,
otherwise unrejected result with incomplete-validation provenance and without
making it reusable checkpoint state.
A lane has these fields: A lane has these fields:
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
@@ -274,17 +316,32 @@ extract:
| --- | --- | --- | --- | | --- | --- | --- | --- |
| **module** | string | none | Required for an object binding. Must be a registered compatible key. | | **module** | string | none | Required for an object binding. Must be a registered compatible key. |
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. | | **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. | | **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
| **options** | object | none | Must satisfy the selected module. | | **options** | object | none | Must satisfy the selected module. |
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. | | **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. | | **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
Omitting **validators** uses the registered chain. **validators: []** selects Omitting **validators** uses the registered chain. **validators: []** selects
an empty chain; a non-empty list replaces the chain in the listed order. an empty chain; a non-empty list replaces the chain in the listed order.
Validator bindings accept only **module**, **llm_profile**, and **options**. Validator bindings accept only **module**, **llm_profile**,
They reject **references**, **retries**, and nested **validators**. Deterministic **structured_output_repair_attempts**, **retries**, and **options**. Their
validators reject an explicit **llm_profile**. Deterministic module bindings **retries** value is a non-negative additional validator-execution budget and
also reject an explicit **llm_profile**. is valid only when the selected validator is LLM-backed. A validator retry
rechecks the same immutable candidate; it never regenerates the producer.
They reject
**validation_policy**, **references**, and nested **validators**. Deterministic
validators reject explicit **llm_profile** and
**structured_output_repair_attempts**.
Deterministic module bindings also reject those explicit fields.
An LLM-backed chunk, extract, merge, or normalize producer with both a
non-empty validator chain and positive **retries** must declare the supported
single-response correction capability. Preparation rejects a configuration
that could require semantic correction from a producer that cannot provide an
exact prior response. A deterministic producer, or an LLM attempt that did
not make a model call, cannot consume a semantic retry after rejection.
The **json** output module accepts optional **include_chunk_map** and The **json** output module accepts optional **include_chunk_map** and
**evidence_context** settings: **evidence_context** settings:
@@ -319,7 +376,8 @@ Unknown outer or nested option fields are rejected, as are incompatible YAML
types. The allowlist remains valid when a run uses lane filtering: a configured types. The allowlist remains valid when a run uses lane filtering: a configured
lane that is not active for that invocation simply contributes no evidence. lane that is not active for that invocation simply contributes no evidence.
Evidence publication is opt-in because it can persist source text and metadata. Evidence publication is opt-in because it can persist source text and metadata.
Its payload contract is [Published Evidence Context](integrations/evidence-context.md). When enabled, it publishes the selected source-unit excerpt defined by the
[Published Evidence Context contract](integrations/evidence-context.md).
## References And Ordered Handoffs ## References And Ordered Handoffs

View File

@@ -0,0 +1,208 @@
# Consuming The Complete D&D Pipeline
Use this workflow when an orchestrator runs the maintained complete D&D
pipeline and consumes its structured JSON artifacts. The generic
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
this guide connects that workflow to the complete D&D configuration, its
Seriatim input, and its artifact inventory.
The [CLI reference](../cli.md), [configuration reference](../config.md),
[run-result receipt](../integrations/run-result.md), and
[published JSON output contract](../integrations/json-output.md) remain the
canonical definitions of those public interfaces.
## Prepare And Validate The Deployment
Start from the maintained
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
artifact handoffs, campaign references, chunk-map publication, and evidence
context.
A deployment must provide its own PromptKit profile and campaign reference
files. Use absolute paths for service and subprocess deployments. In
particular, observe these different resolution rules:
- reference paths in YAML are resolved relative to the Notarius configuration
file; and
- `promptkit.profile_file` is resolved relative to the Notarius process working
directory.
Do not copy the repository example's relative profile path into a deployment
without also controlling that working directory. The complete path and profile
rules are defined in [Configuration](../config.md).
Preflight the deployed configuration before processing sessions and whenever
it changes:
```sh
notarius config validate \
--config /absolute/path/to/notarius.yml \
--pipeline dnd-session
```
Provide credentials through the environment or the documented configuration
mechanism. Do not put credentials in command arguments, generated
configuration, or logs.
## Supply The Transcript
The complete pipeline consumes a Seriatim JSON document. The
[Seriatim input contract](../integrations/seriatim.md) defines its required
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
citations use those segment IDs as source-unit ranges.
When the caller maintains several transcript tiers, use the final trimmed JSON
transcript so extraction operates on the same session content presented to
later consumers. For example, Narratio identifies this implemented artifact as
`narratio.transcript.final_trimmed` and normally stores it at
`transcripts/final.trimmed.json`.
Notarius generates a stable prompt session from the resolved input module and
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
Use that override only when intentionally changing the routing relationship
between invocations; it is not a credential or output identity.
## Run Notarius
Invoke the pipeline with explicit absolute paths and request its
machine-readable receipt:
```sh
notarius run dnd-session \
--config /absolute/path/to/notarius.yml \
--input /absolute/path/to/transcripts/final.trimmed.json \
--output-dir /absolute/path/to/notarius-output \
--json
```
The caller should:
- capture stdout and stderr separately;
- propagate cancellation and impose an operator-appropriate timeout;
- wait for process completion before interpreting stdout; and
- retain stderr for diagnosis without copying secrets or transcript content
into other logs.
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
nonzero exit because a failed receipt write can leave partial bytes. The
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
stream and exit-status contract.
## Discover The Published Bundle
Decode the successful stdout document as a supported run-result schema. For
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate
unknown fields allowed by that version, but reject an unsupported schema
version.
Use the receipt's absolute `output_directory` as the exact run-specific bundle
root. Do not scan the output root for its newest directory, guess a run ID, or
construct a bundle path. Resolve `index_file` beneath `output_directory` and
reject an absolute logical path or any result that escapes the bundle root.
The complete configuration uses the application validation defaults. A caller
that requires fully validated D&D artifacts must also require receipt
`validation_status: approved`; a successful `incomplete` result reflects the
configured validator-failure continuation policy and carries its bounded
validator provenance in `validation_summaries`.
Read `index.json` and locate each requested lane in `output_files` by its exact
`lane_id`. Do not guess a lane filename. Before decoding a payload:
1. resolve its descriptor's relative `file` beneath the bundle root with the
same confinement check;
2. verify the descriptor's media type and schema identity against the linked
artifact contract; and
3. decode the payload according to that contract.
The [published JSON output contract](../integrations/json-output.md) defines
the index and bundle layout. Treat all paths obtained from a decoded external
document as untrusted until confined to their documented root.
## Complete Artifact Inventory
When every configured lane is accepted, the complete example publishes these
lane artifacts:
| Lane ID | Purpose | Canonical contract |
| --- | --- | --- |
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
The JSON encoder always publishes these bundle-management files:
| File | Purpose |
| --- | --- |
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
| `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. |
The complete configuration also requests two pipeline-wide artifacts:
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
chunk metadata; and
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
excerpt containing the union of selected cited source units and the
configured surrounding window.
Discover both from their top-level `index.json` descriptors rather than
treating them as lanes. Evidence context is convenient reading material, not
authoritative provenance; citations in the normalized lane payloads remain the
evidence contract.
Every optional or lane file is published only when its corresponding artifact
is available. A successful process does not guarantee that all configured
lanes were accepted.
## Decide What Counts As Consumer Success
Exit status 0 means Notarius completed the pipeline and published its result
bundle. The receipt or bundle may still report warnings, rejected outputs, or
missing lane descriptors. A downstream consumer must define its own required
artifact set explicitly.
A caller that claims to consume the complete D&D workflow should normally
require all ten lane IDs in the table and verify each descriptor's expected
contract. If any required lane is missing, rejected, or incompatible, fail the
caller's extraction step while retaining the Notarius bundle for diagnosis. A
consumer that needs only a subset may define and document a narrower policy.
Keep the successful receipt with the complete published bundle. Retain
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
required by the caller's provenance, diagnosis, and retention policies. Avoid
selectively copying payload files without also preserving enough index and
manifest information to identify their originating run and contracts.
The transcript, lane artifacts, evidence context, manifest, debug data, and
logs can all contain private campaign information. Apply the same access,
publication, and retention controls used for the source transcript.
## Consumer Checklist
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
- Use absolute configuration, input, output-root, profile, and reference paths
in service deployments.
- Capture stdout and stderr separately and enforce cancellation and timeout.
- Parse stdout only after exit status 0.
- Accept only supported receipt, index, and artifact schema versions while
tolerating permitted unknown fields.
- Use the receipt's `output_directory`; never guess the run directory.
- Confine `index_file` and every descriptor path to the published bundle root.
- Discover lanes by `lane_id` and verify descriptor compatibility before
decoding payloads.
- Enforce an explicit required-lane policy and inspect rejections and warnings.
- Preserve the receipt and sufficient bundle provenance for every retained
artifact.
- Protect all transcript-derived files and diagnostic streams as sensitive
campaign data.

View File

@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
[Published JSON Output contract](../integrations/json-output.md) own the [Published JSON Output contract](../integrations/json-output.md) own the
durable result formats. durable result formats.
For the maintained complete D&D workflow, including its transcript input,
configured lane inventory, and downstream acceptance checklist, see
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
## Run And Check The Process ## Run And Check The Process
Optionally preflight a selected configuration and pipeline before work starts: Optionally preflight a selected configuration and pipeline before work starts:
@@ -55,16 +59,23 @@ contract. The JSON bundle contract links to the available lane contracts.
If `index.json` has an `evidence_context` descriptor, treat it as a If `index.json` has an `evidence_context` descriptor, treat it as a
pipeline-wide artifact rather than a lane entry. Verify its six descriptor pipeline-wide artifact rather than a lane entry. Verify its six descriptor
fields before decoding the linked file according to the [Published Evidence fields before decoding the linked file according to the [Published Evidence
Context contract](../integrations/evidence-context.md). Use each Context contract](../integrations/evidence-context.md). Decode its top-level
`evidence_refs` entry as the citation to source material. Its surrounding source-unit array as a reading excerpt. Obtain authoritative citations and lane
context range and included units explain the citation, but do not widen or provenance from the normalized lane artifacts; the excerpt has neither and its
replace the cited source reference. nearby units do not widen a lane artifact's cited source reference.
A zero exit status may still report rejected outputs, warnings, or absent A zero exit status may still report rejected outputs, warnings, or absent
lanes. The caller decides which lane IDs are required for its own work and lanes. The caller decides which lane IDs are required for its own work and
which are optional; it should make that decision explicitly rather than infer which are optional; it should make that decision explicitly rather than infer
failure from the receipt counts alone. failure from the receipt counts alone.
When complete validation is required, also require receipt
`validation_status: approved` and inspect `validation_summaries`. A successful
run with `validation_status: incomplete` contains a structurally valid result
that advanced after validator execution could not complete under the configured
`warn_continue` policy. It is not reusable checkpoint state and should not be
silently treated as fully reviewed by the caller.
## Preserve Provenance And Handle Data Carefully ## Preserve Provenance And Handle Data Carefully
Keep the receipt with the published `manifest.json`, and retain Keep the receipt with the published `manifest.json`, and retain
@@ -73,5 +84,5 @@ them. Treat the input, output bundle, cache, debug bundle, and captured process
logs as potentially sensitive data. Apply the caller's access controls and logs as potentially sensitive data. Apply the caller's access controls and
retention policy, and avoid copying secrets into arguments, logs, or retention policy, and avoid copying secrets into arguments, logs, or
provenance records. An evidence-context artifact contains source-unit text and provenance records. An evidence-context artifact contains source-unit text and
metadata, and selected lanes can cover most of an input; preserve and share it metadata and can cover most of an input; preserve and share it only when that
only when that source content is authorized for the recipient. source content is authorized for the recipient.

View File

@@ -18,13 +18,14 @@ implemented component map.
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. | | Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. | | Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. | | CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. | | Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. | | Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. | | Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. | | Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. | | LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. | | Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. | | External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. | | Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and the package-local For an existing subsystem, also inspect its focused tests and the package-local

View File

@@ -1,9 +1,11 @@
# Published Evidence Context # Published Evidence Context
This contract defines the optional `source/evidence-context` artifact emitted This contract defines the optional `source/evidence-context` artifact emitted
by the production JSON output. Its configuration is owned by by the production JSON output. It is a selected source-unit excerpt for
[Configuration](../config.md#module-bindings-and-validators); its logical-file convenient reading alongside normalized lane artifacts; it is not a second
discovery is owned by [Published JSON Output](json-output.md). citation or provenance model. Its configuration is owned by
[Configuration](../config.md#module-bindings-and-validators), and its
logical-file discovery is owned by [Published JSON Output](json-output.md).
## Identity And Discovery ## Identity And Discovery
@@ -26,35 +28,13 @@ its absence means evidence publication was not enabled for that bundle.
## Payload ## Payload
The v1 payload is a JSON object with required `source_id`, `source_digest`, The v1 payload is a top-level JSON array of generic source units. There is no
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and wrapper, source-level metadata, context grouping, lane identifier, or evidence
`contexts` are always arrays; an enabled configuration with no accepted direct reference in the payload. An enabled configuration with no contributing
evidence publishes `contexts: []`. accepted evidence publishes `[]`.
```json ```json
{ [
"source_id": "session-alpha",
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
"window_units": 1,
"selected_lanes": ["npc_registry", "spells"],
"contexts": [
{
"context_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 20
},
"evidence_refs": [
{
"lane_id": "spells",
"source_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
}
],
"units": [
{ {
"id": 10, "id": 10,
"kind": "transcript_segment", "kind": "transcript_segment",
@@ -76,41 +56,52 @@ evidence publishes `contexts: []`.
} }
} }
] ]
}
]
}
``` ```
Each context requires a `context_ref` object and `evidence_refs` and `units` Each source unit has required `id`, `kind`, `text`, and self `ref` fields.
arrays. `context_ref` identifies the first and last included unit. Each `ref` contains `source_id`, `start_unit_id`, and `end_unit_id`, and both unit
evidence entry contains a selected `lane_id` and an original `source_ref`. A endpoints identify that unit's `id`. A unit may also contain source-owned
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and `metadata`, an open-ended JSON object. Fixed unit and reference fields are
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject strict: consumers must reject unknown fixed fields, malformed units, invalid
unknown fields; unit metadata may contain application-defined JSON values. self-references, units whose `source_id` differs from other units in the same
excerpt, and a payload that is not the array described here.
## Citations And Context The excerpt preserves each selected unit exactly as represented by the
validated generic source document. It does not add evidence-context-specific
annotations or reshape source-owned metadata.
`evidence_refs` are the authoritative citations. They identify the direct ## Selection And Citations
references emitted by accepted normalized artifacts. `context_ref` and the
units collection include those cited units plus nearby source units selected by
the configured window. They are explanatory context, not widened citations.
Only accepted outputs from the configured lane allowlist contribute. Rejected, The framework obtains direct source references only through typed evidence
failed, absent, and lane-filtered outputs do not contribute. The artifact never projections of accepted normalized artifacts in the configured lane allowlist.
contains raw input bytes, prompts, model responses, auxiliary reference It validates each reference against the current source document, expands its
content, credentials, or filesystem paths. range by `window_units` source-unit positions on each side, clamps at document
boundaries, and takes the union of all expanded ranges. The output contains
each selected source unit once in source-document position order, regardless
of numeric unit IDs. Repeated references, overlapping windows, and citations
from multiple lanes do not duplicate a unit. Rejected, failed, absent,
inactive, and unselected lanes contribute nothing.
## Ordering And Compatibility Normalized lane artifacts remain authoritative for citations and for which lane
cited a range. The excerpt has no lane attribution and must not be used to
reconstruct it. Its included nearby units provide reading context only; they
do not widen any citation in a lane artifact.
The selected lane allowlist is lexical. Contexts and units are in source The excerpt contains at most every generic source unit once. It can therefore
document position order, not numeric unit-ID order. Direct evidence entries equal the complete generic source document when coverage is broad or the
are deterministically ordered by lane and source reference. Overlapping or window is large. No byte-, token-, or compression-size guarantee is made, and
contiguous windows merge, and each source unit appears at most once in the the framework does not truncate the excerpt to meet an arbitrary size limit.
resulting contexts.
## Consumer Responsibilities And Data Handling
The artifact is additive to the JSON bundle and is not a lane payload, The artifact is additive to the JSON bundle and is not a lane payload,
normalized-output count, checkpoint, or generated reference. Consumers that normalized-output count, checkpoint, or generated reference. Consumers that
do not need it must tolerate the absent optional descriptor. Consumers that do do not need it must tolerate an absent descriptor. Consumers that do use it
use it should preserve the artifact and its schema identity with the run should validate the descriptor and payload before use, retain the artifact with
provenance, and should treat its source text and metadata as sensitive durable its schema identity when needed for a run record, and read citations from the
content. corresponding normalized lane artifacts.
The excerpt contains source-unit text and source-owned metadata and is durable
output. Treat it as sensitive source content, apply appropriate access controls
and retention, and do not assume its selected form is materially smaller or
less sensitive than the original input.

View File

@@ -24,7 +24,7 @@ root for the logical discovery described here.
| `warnings.json` | Accepted-output and run warnings. | | `warnings.json` | Accepted-output and run warnings. |
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. | | `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. | | `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. | | `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
JSON files are pretty-printed with a trailing newline. Lane payloads are JSON files are pretty-printed with a trailing newline. Lane payloads are
accepted only when their media type is `application/json`. accepted only when their media type is `application/json`.
@@ -92,7 +92,7 @@ group into the following externally observable summaries:
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` | | Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` | | Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
| Source and references | `source_digests`, `references` | | Source and references | `source_digests`, `references` |
| Published result summaries | `normalized_outputs`, `rejected_outputs` | | Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` | | Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
`references` records provenance such as the target, slot, origin, digest, `references` records provenance such as the target, slot, origin, digest,
@@ -102,6 +102,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
provenance for the plan used by this run; cache records, debug artifacts, and provenance for the plan used by this run; cache records, debug artifacts, and
other operational state are not published as bundle files. other operational state are not published as bundle files.
Each `validation_summaries` entry is a bounded outcome for one producer result.
It has required `status`, `producer_attempt_count`, and `terminal_action`;
the stage and affected step, lane, module, or chunk identity are present when
applicable. `status` is `complete`, `rejected`, or `incomplete`.
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
configured validator order and omit later duplicates. Entries contain no raw
candidate response, correction guidance, validator diagnostic message, or
artifact payload. The same shape may appear as `validation` on an affected
rejection entry.
When present, `metadata.session_id` is the effective non-secret routing When present, `metadata.session_id` is the effective non-secret routing
correlation identifier used for the run. It can be visible to providers and is correlation identifier used for the run. It can be visible to providers and is
not a substitute for a cache or checkpoint identity. Its generation and not a substitute for a cache or checkpoint identity. Its generation and
@@ -127,7 +137,10 @@ distinct even when their profile, provider, and model are otherwise equal.
`rejected.json` is always an object with a `rejected` array. Each entry has `rejected.json` is always an object with a `rejected` array. Each entry has
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`, required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and `chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` are present only when applicable. `diagnostic_artifact_path` are present only when applicable. An entry may also
contain the bounded `validation` summary described above; the existing singular
validator and reason fields remain the first configured rejection for
compatibility.
`warnings.json` is always an object with a `warnings` array. Each warning has `warnings.json` is always an object with a `warnings` array. Each warning has
`reason_code` and `message`; `scope` is optional. Both arrays are empty when `reason_code` and `message`; `scope` is optional. Both arrays are empty when

View File

@@ -1,11 +1,11 @@
# PromptKit Integration # PromptKit Integration
Notarius pins Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0) [`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
as its in-process prompt engine. The upstream as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md) [Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md) [format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
owns prompt, profile, and schema file contracts. owns prompt, profile, and schema file contracts.
## Supported Boundary ## Supported Boundary
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
- construct an `Engine` with filesystem-backed prompt, schema, and optional - construct an `Engine` with filesystem-backed prompt, schema, and optional
operator and application-fallback profile sources; operator and application-fallback profile sources;
- prepare one frozen execution from a `RunRequest` with named inline artifacts, - prepare one frozen execution from a `RunRequest` with named inline artifacts,
variables, a direct session ID, prompt identity, and profile selection, then variables, a direct session ID, prompt identity, profile selection, and
optional appended rendered messages, then
record credential-redacted details and run that exact execution; record credential-redacted details and run that exact execution;
- return rendered debug material, validated structured output, selected - return rendered debug material, validated structured output, selected
profile, backend, effective model metadata, and token usage; profile, backend, effective model metadata, and token usage;
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
admission exhaustion through `ErrCapacityExceeded`. admission exhaustion through `ErrCapacityExceeded`.
The pinned The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go) [`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
owns the registration and backend-capacity contract. owns the registration and backend-capacity contract.
For one completion, the adapter calls `PrepareExecution`, takes a For one completion, the adapter calls `PrepareExecution`, takes a
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
field, which is authoritative for provider session behavior. It also retains field, which is authoritative for provider session behavior. It also retains
the same value as the `session_id` prompt variable for maintained prompt the same value as the `session_id` prompt variable for maintained prompt
compatibility. The generated identifier is 76 ASCII characters, within compatibility. The generated identifier is 76 ASCII characters, within
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
correlation identifiers and may be exposed to providers and provider correlation identifiers and may be exposed to providers and provider
observability. The CLI contract owns generation and override behavior. observability. The CLI contract owns generation and override behavior.
@@ -84,7 +85,39 @@ configuration and deployment workflow are defined in
[Configuration](../config.md#promptkit-profiles) and [Configuration](../config.md#promptkit-profiles) and
[Operations](../operations.md#promptkit-profile-deployment). [Operations](../operations.md#promptkit-profile-deployment).
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source, PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403.
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
`assistant` text-chat roles after normalizing case and surrounding whitespace.
Maintained Notarius prompt definitions use only `system` and `user`.
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
exactly two messages in order: the latest validated producer response with
role `assistant`, then deterministic validation guidance with role `user`.
It never exposes a general caller-selected role API, accumulates earlier
correction turns, or changes the ordinary prompt prefix. Ordinary requests
leave appended messages unset.
PromptKit preserves supplied content but does not own Notarius's correction
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
1,114,112 bytes) before preparing the request. The transport-neutral
application contract owns defensive copying and these limits. Default request
and terminal summaries retain only safe counts, digests, identities, and usage;
complete appended messages remain limited to the explicitly requested detailed
debug trace.
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
profile catalogs from independently versioned transitive modules. Notarius
does not import or register either catalog; PromptKit retains catalog source,
identity, precedence, credential, and capacity ownership.
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter, implicitly supported. A dependency upgrade requires reviewing the adapter,
@@ -97,6 +130,10 @@ pinned upstream documentation.
module assets, maps its transport-neutral completion contract, prepares and module assets, maps its transport-neutral completion contract, prepares and
executes requests, validates output, records provenance, captures debug executes requests, validates output, records provenance, captures debug
material, redacts errors, and preserves timeout ownership. material, redacts errors, and preserves timeout ownership.
PromptKit provider error details do not cross the ordinary completion boundary.
Notarius exposes a provider-neutral generation category and optional status;
redacted provider details are retained only in requested debug material.
[D&D Module Internals](../internal/dnd.md) owns the embedded [D&D Module Internals](../internal/dnd.md) owns the embedded
`dnd-extraction` fallback profile and the maintained D&D prompt defaults. `dnd-extraction` fallback profile and the maintained D&D prompt defaults.
[Configuration](../config.md#promptkit-profiles) defines how a Notarius [Configuration](../config.md#promptkit-profiles) defines how a Notarius
@@ -105,4 +142,7 @@ the conventional local backend.
PromptKit API or format changes outside this boundary are not implicitly PromptKit API or format changes outside this boundary are not implicitly
supported. Updating the pinned version requires reviewing the adapter and supported. Updating the pinned version requires reviewing the adapter and
profile/configuration contracts against the upstream documentation. profile/configuration contracts against the upstream documentation. Maintained
production prompts use PromptKit's bounded structural-repair contract; their
current declaration is one additional repair attempt. Notarius retains the
transport-neutral boundary and does not expose PromptKit types to modules.

View File

@@ -22,11 +22,15 @@ The current schema version is `notarius.run-result.v1`.
| `rejected_output_count` | Yes | Number of recorded rejected outputs. | | `rejected_output_count` | Yes | Number of recorded rejected outputs. |
| `warning_count` | Yes | Number of final run warnings. | | `warning_count` | Yes | Number of final run warnings. |
| `validation_status` | Yes | The final run manifest validation status. | | `validation_status` | Yes | The final run manifest validation status. |
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. | | `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
For the production `json` output module, `index_file` is present only when the For the production `json` output module, `index_file` is present only when the
completed run returned exactly one logical output file named `index.json`. completed run returned exactly one logical output file named `index.json`.
For another output module, its absence does not indicate a failed run. For another output module, its absence does not indicate a failed run.
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
means one or more otherwise accepted results advanced under validator-failure
`warn_continue`.
```json ```json
{ {
@@ -38,7 +42,17 @@ For another output module, its absence does not indicate a failed run.
"normalized_output_count": 6, "normalized_output_count": 6,
"rejected_output_count": 2, "rejected_output_count": 2,
"warning_count": 1, "warning_count": 1,
"validation_status": "rejected" "validation_status": "incomplete",
"validation_summaries": [
{
"stage": "extract",
"lane_id": "spells",
"status": "incomplete",
"incomplete_validators": ["dnd/spells/source_refs"],
"producer_attempt_count": 1,
"terminal_action": "warn_continue"
}
]
} }
``` ```
@@ -48,8 +62,11 @@ For another output module, its absence does not indicate a failed run.
paths. They identify the paths used by Notarius and do not resolve symlinks. paths. They identify the paths used by Notarius and do not resolve symlinks.
`output_directory` is the run-specific bundle, not the configured output root. `output_directory` is the run-specific bundle, not the configured output root.
The receipt is a summary and discovery document. It does not contain lane The receipt is a summary and discovery document. Its optional validation
descriptors, payloads, manifest data, rejections, warnings, or file contents. summaries contain only stable status, identity, validator names, reason codes,
attempt counts, and terminal actions. It does not contain lane descriptors,
payloads, manifest payloads, rejection messages, warnings, raw model responses,
correction guidance, or file contents.
For the production JSON output, resolve `index_file` beneath For the production JSON output, resolve `index_file` beneath
`output_directory`, reject path escapes, and use the `output_directory`, reject path escapes, and use the
[Published JSON Output contract](json-output.md) to discover logical files and [Published JSON Output contract](json-output.md) to discover logical files and

View File

@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
## Dispatch And Configuration Handoff ## Dispatch And Configuration Handoff
The root dispatcher handles help, configuration validation, pipeline listing, The root dispatcher handles help, version reporting, configuration validation,
and a pipeline run. It normalizes injectable options before dispatch so that a pipeline listing, and a pipeline run. Version reporting resolves build
missing production dependency fails as a command error rather than reaching information through `internal/buildinfo` before production composition, so the
execution. diagnostic remains available without configuration or runtime collaborators.
The public syntax, streams, exit classes, and version semantics are defined by
the [CLI reference](../cli.md). Other root commands normalize injectable
options before dispatch so that a missing production dependency fails as a
command error rather than reaching execution.
Commands that need configuration use one shared loader. The CLI discovers the Commands that need configuration use one shared loader. The CLI discovers the
file, parses it through **internal/core/config**, starts from defaults, applies file, parses it through **internal/core/config**, starts from defaults, applies

View File

@@ -74,10 +74,19 @@ profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by public field definitions and precedence are owned by
[Configuration](../config.md#pipelines). [Configuration](../config.md#pipelines).
The resolver retains configured `validation_policy` overrides and derives one
detached concrete terminal policy for the chunk producer and every lane's
extract, merge, and normalize producers. That field-by-field inheritance is
complete before preparation, and the effective values contribute to pipeline
and checkpoint identity; execution does not interpret configuration defaults.
The framework resolver supplies defaults, selects lanes, resolves validator The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting options, and returns the fixed ordered pipeline shape. Positive validator retry
**EffectiveConfig** retains the selected ID, requested selection and reference budgets require an LLM-backed selected validator; deterministic validators are
rejected during resolution. Eligible LLM-backed producer specifications also
contribute their declared correction protocol to the resolved metadata. The
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
changes, a clone of the input configuration, and the resolved pipeline. changes, a clone of the input configuration, and the resolved pipeline.
Callers may therefore retain or modify their input slices and maps without Callers may therefore retain or modify their input slices and maps without
changing the resolved result, and later consumers cannot mutate the original changing the resolved result, and later consumers cannot mutate the original
@@ -94,8 +103,8 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
The framework assigns the resolved pipeline a deterministic SHA-256 digest The framework assigns the resolved pipeline a deterministic SHA-256 digest
after defaults, lane selection, module bindings, reference bindings, validator after defaults, lane selection, module bindings, reference bindings, validator
chains, effective LLM profiles, and artifact schema identity have been chains, selected correction protocols, effective LLM profiles, and artifact
resolved. The digest excludes schema identity have been resolved. The digest excludes
its own stored value. It identifies resolved composition rather than raw YAML its own stored value. It identifies resolved composition rather than raw YAML
bytes, a debug payload, or all runtime state. The CLI records it as invocation bytes, a debug payload, or all runtime state. The CLI records it as invocation
provenance before execution; cache and checkpoint identity have additional provenance before execution; cache and checkpoint identity have additional

View File

@@ -33,7 +33,9 @@ typed builder. Scene chunking, every extractor, and NPC, location, and item-regi
normalization are registered as `llm_backed`; the remaining current D&D mergers normalization are registered as `llm_backed`; the remaining current D&D mergers
and normalizers are `deterministic`. The metadata is available to catalog inspection and and normalizers are `deterministic`. The metadata is available to catalog inspection and
resolved-pipeline debug data and determines which selected bindings inherit the resolved-pipeline debug data and determines which selected bindings inherit the
pipeline profile. Configuration remains the canonical owner of the exact keys, pipeline profile. The registry normalizers use `single_response_v1`, forwarding
corrections to their reconciliation completion and retaining the accepted raw
proposal only as an owned model candidate. Configuration remains the canonical owner of the exact keys,
profile precedence, and validator order. profile precedence, and validator order.
Private structured-LLM response schemas are deliberately minimal. They reject Private structured-LLM response schemas are deliberately minimal. They reject
@@ -42,6 +44,13 @@ unknown fields, while preserving semantic candidates for deterministic
validation. Do not promote a private response envelope into a durable schema; validation. Do not promote a private response envelope into a durable schema;
the contracts above define durable data. the contracts above define durable data.
A D&D producer that declares `single_response_v1` forwards any supplied
semantic correction to its structured completion and returns an owned copy of
that completion's exact validated raw response as its model candidate. It does
not serialize normalized artifacts to create that candidate, so deterministic
identity, evidence, warning, and durable-schema behavior remains separate from
the model transport material.
## Prompt Construction ## Prompt Construction
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
@@ -125,6 +134,13 @@ relatedness validators report advisory evidence concerns. The configured order
is documented in is documented in
[Configuration](../config.md#production-validator-keys-and-default-chains). [Configuration](../config.md#production-validator-keys-and-default-chains).
Every D&D rejection describes the correction in transcript-grounded domain
terms, using contextual names, artifact fields, and source segment ranges when
useful. The guidance must not ask the model to reproduce durable entity IDs,
hashes, validator module keys, or reason codes. Those identifiers remain in
ordinary validation provenance; only the actionable semantic guidance is
eligible for the correction prompt.
Enemy-event extraction additionally rejects a second `engaged` observation for Enemy-event extraction additionally rejects a second `engaged` observation for
the same comparison identity within one scene-scoped result. Normalization may the same comparison identity within one scene-scoped result. Normalization may
combine results from distinct scenes, so it intentionally does not apply that combine results from distinct scenes, so it intentionally does not apply that

View File

@@ -42,6 +42,22 @@ observability. The adapter returns PromptKits validated raw bytes rather than
re-encoding the decoded target. An empty optional material is represented as re-encoding the decoded target. An empty optional material is represented as
one space so its named input is retained by PromptKit. one space so its named input is retained by PromptKit.
When a request includes semantic correction material, the adapter validates and
defensively copies it before preparation, then appends exactly two messages
after the ordinarily rendered prompt: the prior response as an assistant
message and the correction guidance as a user message. Requests without a
correction do not add messages or introduce caller roles. Ordinary request
summaries record correction byte counts and digests only; complete messages are
available solely in an explicitly requested debug trace.
The adapter leaves the ordinary rendered message prefix, named inputs,
variables, session, profile, execution overrides, prepared-execution path, and
PromptKit repair policy unchanged for a corrected request. It never imports a
PromptKit message type into a module or pipeline contract. PromptKit reports
actual structural repair count and cumulative token usage per completion; the
pipeline's safe terminal debug record projects those values without copying
message content.
Client construction may also receive a run-wide reasoning-effort override from Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer creates a fresh PromptKit execution override for each request: a nil pointer
@@ -75,7 +91,7 @@ backend membership as runtime without performing generation. Fallback assets
are mounted only when at least one source is registered. The production D&D are mounted only when at least one source is registered. The production D&D
registrar contributes its `dnd-extraction` fallback, and the maintained D&D registrar contributes its `dnd-extraction` fallback, and the maintained D&D
prompts select that logical ID by default. PromptKit owns source precedence and prompts select that logical ID by default. PromptKit owns source precedence and
profile parsing: an operator-provided matching profile takes precedence over a profile parsing and inheritance: an operator-provided matching profile takes precedence over a
fallback profile without Notarius merging either document. fallback profile without Notarius merging either document.
When the registration is absent, a profile selecting `backend: local` fails When the registration is absent, a profile selecting `backend: local` fails
inspection instead of falling back to a built-in or endpoint-only target. inspection instead of falling back to a built-in or endpoint-only target.
@@ -210,13 +226,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
the pipeline's existing binding attempt policy sees the operational error and the pipeline's existing binding attempt policy sees the operational error and
decides whether to rerun the complete operation. decides whether to rerun the complete operation.
Prompt-declared repair is executed within PromptKits structured-output flow. PromptKit executes structural repair within its structured-output flow. The
The current production D&D prompt manifests set repair attempts to zero. That maintained production prompt manifests declare one additional repair attempt.
setting does not replace pipeline retry behavior: a bindings configured retry When a resolved binding supplies a repair value, the adapter inspects the
count reruns its stage attempt after an error or rejection, and an exhausted prompt, copies its complete output contract, changes only the repair limit, and
rejection is a recorded output rather than a provider error. The pipeline owns passes that complete replacement contract to PromptKit. This preserves the
attempt lifecycle, validation chains, and retry diagnostics; see prompt's output format, validation mode, schema, and provider structured-output
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the settings.
A successful repair is an ordinary successful completion, not a warning. The
adapter reports PromptKit's actual repair count and its cumulative usage
directly, without adding the initial and corrective counts again. Debug prompt
material records the configured complete contract; debug response material
records the repaired response and actual validation result. If the repair
budget is exhausted, the adapter retains the final raw bytes and debug material
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
or corrective call remain provider-neutral operational errors with the same
redaction boundary.
Structural repair does not replace pipeline retry behavior: a binding's
configured retry count reruns its complete stage attempt after an operational
or structural error, module-requested retry, or actionable semantic rejection.
The pipeline owns attempt lifecycle, validation chains, and retry diagnostics;
see [Pipeline Internals](pipeline.md#validation-retries-and-output) and the
[binding reference](../config.md#module-bindings-and-validators). [binding reference](../config.md#module-bindings-and-validators).
## Timeout Ownership ## Timeout Ownership
@@ -247,6 +279,13 @@ surfaced; when the completion already failed, its call error remains the
result. Debug-bundle location, retention, and handling are operational concerns result. Debug-bundle location, retention, and handling are operational concerns
documented in [Operations](../operations.md#debug-bundles). documented in [Operations](../operations.md#debug-bundles).
The attempt-terminal summary is a separate safe trace record: it contains
attempt kinds, validator outcome counts and reason codes, effective policy,
terminal action, and repair/usage references. It excludes raw assistant
responses and correction text. Those values can appear only in the explicitly
requested detailed prompt and response artifacts, which require sensitive-data
handling.
Run manifests receive selected profile summaries, including optional effective Run manifests receive selected profile summaries, including optional effective
backend and reasoning provenance, and component identities—not prompt, schema, backend and reasoning provenance, and component identities—not prompt, schema,
source, reference, or response content. The published field semantics belong source, reference, or response content. The published field semantics belong
@@ -256,6 +295,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
available to other runtime collaborators; it does not make prompt or response available to other runtime collaborators; it does not make prompt or response
contents safe for general logging. contents safe for general logging.
Generation failures expose an application-owned category and optional HTTP
status. Provider code, type, and message remain debug-only, after redaction.
## Failure Boundaries ## Failure Boundaries
- Construction fails for missing asset registries, mutually exclusive profile - Construction fails for missing asset registries, mutually exclusive profile

View File

@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
binding-specific profile on a deterministic module. The user-facing precedence binding-specific profile on a deterministic module. The user-facing precedence
contract belongs in [Configuration](../config.md#pipelines). contract belongs in [Configuration](../config.md#pipelines).
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
declare correction protocol `single_response_v1`. That declaration is a
promise that the implementation accepts one attempt-local semantic correction
and returns an owned copy of the exact one model response that directly
controlled the candidate. It must forward correction only to its structured
completion request; it must not manufacture prior-response material by
serializing a normalized artifact or expose opaque application IDs. Input,
output, validator, and deterministic specs cannot declare the protocol.
Implementations that accept options must provide both an option validator and Implementations that accept options must provide both an option validator and
a builder. The validator is used while resolving configuration; the builder a builder. The validator is used while resolving configuration; the builder
decodes the same options and constructs the implementation from the prepared decodes the same options and constructs the implementation from the prepared
@@ -36,13 +45,23 @@ they need, register each leaf implementation, and add any family-owned assets
or default validator chains. They return contextual errors so production or default validator chains. They return contextual errors so production
composition fails at startup rather than at the first run. composition fails at startup rather than at the first run.
A validator that returns a completed rejection must supply two separate
bounded values: a stable `ReasonCode` for provenance and actionable
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
and the constraints on one complete corrected replacement. It must not contain
validator keys, diagnostic paths, opaque application IDs, or other internal
identifiers. An operator-facing `Message` may explain the same event, but the
framework never copies it into a model request. Missing or invalid guidance is
a validator contract failure.
An artifact family can register an optional typed evidence projector alongside An artifact family can register an optional typed evidence projector alongside
its codec. The projector returns defensive copies of the artifact's direct its codec. The projector returns defensive copies of the artifact's direct
generic source references and must use the codec's exact Go type. It does not generic source references and must use the codec's exact Go type. It does not
interpret surrounding context or publish files; the pipeline validates the interpret surrounding context or publish files; the pipeline validates the
capability during preparation and the output boundary owns publication. See capability during preparation and the output boundary owns publication. See
the [Published Evidence Context contract](../integrations/evidence-context.md) the [Published Evidence Context contract](../integrations/evidence-context.md)
for the durable result. for the durable source-unit excerpt. Lane artifacts retain citation and lane
provenance; the framework does not add either to that published excerpt.
An artifact family is broader than a module: it owns the cohesive domain An artifact family is broader than a module: it owns the cohesive domain
feature across its artifact type, codec, stage modules, validators, prompt feature across its artifact type, codec, stage modules, validators, prompt
@@ -90,6 +109,12 @@ combined-material bound preserves the deterministic result under the family's
fallback policy. Provider, transport, cancellation, and context-construction fallback policy. Provider, transport, cancellation, and context-construction
failures remain execution errors. failures remain execution errors.
When the engine actually makes a proposal call, its typed result carries the
owned exact proposal response under the same correction contract as other
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
model candidate, so a later rejection applies terminal policy without spending
an ineffective semantic retry.
The core supplies a conservative generic prompt and the single private The core supplies a conservative generic prompt and the single private
response schema. A domain prompt may substitute its semantic instructions but response schema. A domain prompt may substitute its semantic instructions but
mounts the core-owned protocol and candidate/transcript presentation assets. mounts the core-owned protocol and candidate/transcript presentation assets.
@@ -110,9 +135,13 @@ its domain prompt.
3. Implement strict option decoding, construction, and the typed stage 3. Implement strict option decoding, construction, and the typed stage
interface. Preserve caller ownership: do not retain mutable request data interface. Preserve caller ownership: do not retain mutable request data
and return defensive copies where an implementation exposes stored data. and return defensive copies where an implementation exposes stored data.
If declaring correction capability, forward the request correction and
retain only the exact validated response that controlled the result.
4. Register the module through its typed registry helper and add it to the 4. Register the module through its typed registry helper and add it to the
owning family registrar. Add a default validator chain only when that owning family registrar. Add a default validator chain only when that
family owns the behavior; otherwise require an explicit compatible chain. family owns the behavior; otherwise require an explicit compatible chain.
Every rejection path in a validator must provide actionable correction
guidance while retaining its stable internal reason code.
5. Update the selectable-key and chain reference in 5. Update the selectable-key and chain reference in
[Configuration](../config.md#production-module-keys), the applicable [Configuration](../config.md#production-module-keys), the applicable
integration contract, and focused tests. Keep the configuration document integration contract, and focused tests. Keep the configuration document

View File

@@ -25,6 +25,7 @@ physical state roots.
| Area | Implemented owners | Responsibility | | Area | Implemented owners | Responsibility |
| --- | --- | --- | | --- | --- | --- |
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. | | Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. | | Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. | | Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. | | Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |

View File

@@ -32,13 +32,26 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
It normalizes the pipeline and lane identities, applies stage defaults, selects It normalizes the pipeline and lane identities, applies stage defaults, selects
requested lanes where that is supported, resolves validator chains, checks requested lanes where that is supported, resolves validator chains, checks
module capabilities and typed artifact compatibility, validates options, and module capabilities and typed artifact compatibility, validates options, and
assigns a deterministic resolved-composition digest. The resolved pipeline assigns a deterministic resolved-composition digest. A correction protocol is
contains bindings and declared reference targets, not external reference bytes. selected from each eligible LLM-backed producer specification and becomes part
of that resolved identity; only `single_response_v1` is currently supported.
Preparation rejects an LLM-backed producer that combines a non-empty validator
chain with positive producer retries unless it declares that protocol. Producers
without validators or without retries remain valid without correction support.
The resolved pipeline contains bindings and declared reference targets, not
external reference bytes.
After selection, the resolver applies command, binding, and pipeline profile After selection, the resolver applies command, binding, and pipeline profile
precedence to LLM-backed bindings and validators only; prompt defaults remain precedence to LLM-backed bindings and validators only; prompt defaults remain
an empty resolved binding profile. Deterministic bindings remain profile-free. an empty resolved binding profile. It resolves structural output repair
These effective values are part of the digest, so execution and checkpoint separately: a binding's `structured_output_repair_attempts` value wins, then a
consumers do not repeat profile inheritance. pipeline value applies to LLM-backed bindings and validators, and omission
leaves the prompt-owned policy intact. An explicit repair value on a
deterministic binding is rejected. Resolved bindings own copied repair values,
and these effective values are part of the digest, so execution and checkpoint
consumers do not repeat profile inheritance or configuration resolution.
Each LLM request receives its own copy of that resolved value. PromptKit spends
it only for structural correction inside one completion; the runner's binding
retry policy remains the separate outer budget for complete stage attempts.
Configuration resolution supplies the selected profile and catalog; see Configuration resolution supplies the selected profile and catalog; see
[Configuration Internals](configuration.md). [Configuration Internals](configuration.md).
@@ -52,9 +65,11 @@ remains declared but has no bytes until its producing step completes.
Preparation is the construction boundary. It validates the resolved shape and Preparation is the construction boundary. It validates the resolved shape and
registry set, clones the resolved data, then constructs the input adapter, registry set, clones the resolved data, then constructs the input adapter,
chunker, stage-local validators, every typed lane, and output encoder. Each chunker, stage-local validators, every typed lane, and output encoder. The
registered builder receives its own cloned build request immediately before its prepared producer metadata preserves each selected correction protocol, and
module-owned code runs. Preparation also collects stable checkpoint the resolved digest carrying that metadata participates in checkpoint identity.
Each registered builder receives its own cloned build request immediately
before its module-owned code runs. Preparation also collects stable checkpoint
fingerprints. Missing registrations, incompatible typed entries, nil fingerprints. Missing registrations, incompatible typed entries, nil
implementations, and constructor failures are reported before source parsing implementations, and constructor failures are reported before source parsing
or any stage operation begins. or any stage operation begins.
@@ -121,14 +136,45 @@ their target: chunks, codec-decoded typed candidates, or serialized codec
bytes. Each typed validator receives a newly decoded value from the one bytes. Each typed validator receives a newly decoded value from the one
candidate serialization for that attempt, while serialized validators receive candidate serialization for that attempt, while serialized validators receive
separately owned representation bytes and schema metadata. They may approve, separately owned representation bytes and schema metadata. They may approve,
approve with warnings, reject, or fail. A rejection is an ordinary pipeline approve with warnings, reject, fail, or be skipped when a runtime prerequisite
result; a validator error is a framework error. is unavailable. The shared executor settles every configured validator in
order. A failed LLM-backed validator retries only itself against the same
immutable candidate; it does not regenerate the producer or alter the
validator request. Rejections stop that validator, while other configured
validators still run. The executor retains ordered results, bounded
deduplicated correction guidance from rejections, and only the final exhausted
failure outcome for each validator. The correction builder keeps first
occurrence order, omits internal reason codes, validator names, and operator
messages, and requests one complete replacement. Missing guidance or an
oversized aggregate is a framework contract error; guidance is never inferred
or truncated.
The runner applies the binding's retry policy around a stage operation and its The runner applies the binding's retry policy around a stage operation and its
complete validation chain. It preserves warnings only from the final accepted complete validation chain. It preserves warnings only from the final accepted
or rejected attempt. Cancellation stops retries. Normalizer-specific retry or rejected attempt, plus one fixed warning per validator whose execution
directives consume this same budget and validate any final safe fallback through budget was exhausted under `warn_continue`. Cancellation stops retries.
the normalizer chain. Normalizer-specific retry directives consume this same budget and validate any
final safe fallback through the normalizer chain.
The artifact-neutral producer-attempt state machine owns that shared budget,
attempt provenance, semantic-correction material, and terminal-policy
selection. It accepts producer and complete-validation closures, so artifact
materialization, cache handling, checkpoints, and debug output stay at the
operation boundary. It distinguishes operational, structural, module-requested,
and semantic retries. A semantic retry is available only for a valid latest
`single_response_v1` candidate; a deterministic or no-model rejection instead
settles the semantic policy immediately. Structural-output errors alone use the
structural policy, and validation failure without rejection settles the
validator-failure policy without regenerating the producer.
Chunk planning uses this state machine for generated plans. A rejected or
validation-incomplete automatic cache hit is not model material and therefore
falls through to a fresh initial generation at producer attempt one; it neither
receives a correction, consumes retry budget, promotes cached-candidate
warnings, nor overwrites the stored record. An incomplete cache validation
under `fail_run` terminates instead. Only a newly generated, completely
validated plan is published to the chunk-plan store. Rejected plans never
advance, and validation-incomplete plans remain unpublishable.
After terminal lane work, the runner assembles manifest provenance, normalized After terminal lane work, the runner assembles manifest provenance, normalized
artifacts, rejections, warnings, and an optional accepted chunk map. When an artifacts, rejections, warnings, and an optional accepted chunk map. When an
@@ -142,6 +188,15 @@ The CLI publishes those files only after the runner returns without a framework
error. Logical file names and schemas are defined by the [output integration error. Logical file names and schemas are defined by the [output integration
contracts](../integrations/). contracts](../integrations/).
For every completed producer disposition, the runner projects one bounded
validation summary to the manifest, the affected rejection when present, and
the CLI result receipt. The summary records status, configured-order rejecting
validators and reason codes, incomplete validators, producer-attempt count,
and terminal action. It contains no operator message, correction guidance, or
model response. `complete`, `rejected`, and `incomplete` describe the final
candidate disposition; a run-level `incomplete` status indicates at least one
current-run output advanced under `warn_continue`.
## Checkpoint And Debug Hooks ## Checkpoint And Debug Hooks
The runner receives checkpoint and debug interfaces rather than roots. It The runner receives checkpoint and debug interfaces rather than roots. It
@@ -151,6 +206,20 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
Selective recomputation can require a canonical accepted normalized predecessor Selective recomputation can require a canonical accepted normalized predecessor
before a dependent lane starts. before a dependent lane starts.
The runner writes successful checkpoint artifacts only after complete accepted
validation. Chunk plans follow the same rule for publication. A rejection,
invalid structured response, or incomplete validation is never reusable state;
the current run may still hand off an otherwise valid `warn_continue` result
according to its terminal policy. The runner carries private reuse eligibility
through extract, merge, normalize, and generated-reference handoff. Any stage
derived from incomplete validation skips both checkpoint lookup and all
checkpoint publication even when that stage's own validation completes.
External references and fully validated generated references remain eligible.
Attempt debug records retain safe kind,
validator, repair-usage, policy, and terminal-decision provenance. Full
assistant and correction content remains confined to the requested detailed
LLM trace.
Debug recording is attempt-scoped and application-owned. A failure to persist Debug recording is attempt-scoped and application-owned. A failure to persist
required debug data is a framework error. State roots, persistence, reason-code required debug data is a framework error. State roots, persistence, reason-code
meanings, resume, and cleanup are intentionally owned by meanings, resume, and cleanup are intentionally owned by

View File

@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers [Configuration](config.md) owns fields, defaults, and precedence. Maintainers
who need implementation mechanics should read [Run State Internals](internal/state.md). who need implementation mechanics should read [Run State Internals](internal/state.md).
## Source Deployment
Linux is the supported deployment platform. Install a pinned source release
with the Go version declared in `go.mod` (currently Go 1.25.5):
~~~sh
GOWORK=off go install \
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
~~~
Pin the exact tag in deployment automation rather than following a branch.
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
The maintainer publication process, including tag guards and verification,
belongs to [Source Releases](release.md). macOS builds are best-effort for
development, and Windows is unsupported.
## State Surfaces ## State Surfaces
Each run can use independent roots with different retention and access-control Each run can use independent roots with different retention and access-control
@@ -72,6 +89,9 @@ provider call or credentials:
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
~~~ ~~~
An unset optional `api_key_env` reaches the provider without authorization and
may receive a 401 or 403 response.
Profile paths are currently resolved from the process working directory, not Profile paths are currently resolved from the process working directory, not
from the configuration file. The complete example's from the configuration file. The complete example's
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root `./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
@@ -89,6 +109,33 @@ On success, the command reports the output bundle path. A warning-bearing run
still succeeds and reports its warning count on standard error. Errors and still succeeds and reports its warning count on standard error. Errors and
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses). their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
## Validation Retries And Terminal Outcomes
Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from
PromptKit's structural-repair calls inside one completion and from an
LLM-backed validator's own retry budget. A semantic correction rebuilds the
ordinary producer request and supplies only the latest rejected model response
plus aggregated validator guidance; it is not a conversation replay.
After the applicable budgets are exhausted, the resolved
[`validation_policy`](config.md#pipelines) determines the result. Structural
failure and semantic rejection normally fail the run; an explicit
`reject_output` records a rejection and allows unrelated work to finish. A
validator execution failure normally uses `warn_continue`, which keeps an
otherwise accepted result in the current run with `incomplete` validation
provenance. It emits one bounded warning for every validator whose execution
budget was exhausted. A corrected result that later passes validation does not
retain abandoned-attempt warnings.
Treat a successful process exit as a completed run, not as proof that every
candidate was fully validated. Inspect the receipt's `validation_status`,
`validation_summaries`, rejection count, and warning count when an orchestrator
requires complete validation. The durable fields and their meanings are owned
by the [run-result receipt](integrations/run-result.md) and
[published JSON output contract](integrations/json-output.md).
## Output Bundles ## Output Bundles
Each successful run receives a generated safe run identifier and writes beneath: Each successful run receives a generated safe run identifier and writes beneath:
@@ -110,9 +157,9 @@ are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
[evidence context](integrations/evidence-context.md) contains source-unit text [evidence context](integrations/evidence-context.md) contains source-unit text
and metadata. It is not a cache or debug artifact: retain it with the output and metadata. It is not a cache or debug artifact: retain it with the output
bundle only for as long as consumers need it, and apply source-content access bundle only for as long as consumers need it, and apply source-content access
controls to the entire bundle. Selected lanes may collectively cite most of a controls to the entire bundle. Its selected source-unit excerpt may include
transcript, so a broad allowlist can make the evidence artifact nearly as every source unit once when coverage is broad or its configured window is
sensitive and large as the source itself. large, so do not assume a byte or token reduction or reduced sensitivity.
## Chunk-Plan Cache ## Chunk-Plan Cache
@@ -138,7 +185,9 @@ The configured cache mode controls one invocation:
A reused plan is still materialized and validated against the current source. A reused plan is still materialized and validated against the current source.
If a prior plan no longer gives acceptable results, use a refresh run rather If a prior plan no longer gives acceptable results, use a refresh run rather
than editing cache files. Deleting a plan is recoverable but can repeat costly than editing cache files. Deleting a plan is recoverable but can repeat costly
chunking work. chunking work. A plan accepted only under incomplete validation is not
published, and a rejected cache hit falls through to ordinary generation rather
than becoming a correction candidate.
## Checkpoint Recording, Resume, And Recompute ## Checkpoint Recording, Resume, And Recompute
@@ -163,6 +212,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
runtime identities, so checkpoints created under one state are not reused by runtime identities, so checkpoints created under one state are not reused by
either of the others. either of the others.
Only accepted, completely validated chunk, extract, merge, and normalize
results are checkpointed for reuse. Rejected, structurally invalid, and
validation-incomplete producer results remain non-reusable, even when a
`warn_continue` result advanced during its original run. A resumed invocation
therefore reruns that producer rather than treating degraded state as accepted.
Checkpoint state is confined below an identity-specific path: Checkpoint state is confined below an identity-specific path:
~~~ ~~~
@@ -217,13 +272,16 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
~~~ ~~~
The summary contains redacted invocation and resolution information plus run, The summary contains redacted invocation and resolution information plus run,
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace warning, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
contains allowlisted application diagnostic records and can include source or terminal records contain bounded attempt kinds, validator outcomes, policy,
derived application data. Neither surface is a cache input. Do not treat a decision, PromptKit repair count, and usage; they do not contain assistant
debug bundle as safe to share merely because its configuration summary is responses or complete correction messages. The trace contains allowlisted
redacted. Invocation metadata omits reasoning effort when it is inherited, application diagnostic records and can include source, model, and correction
records the replacement value when one is supplied, and records an empty value content. Neither surface is a cache input. Do not treat a debug bundle as safe
when inherited reasoning was explicitly cleared. to share merely because its configuration summary is redacted. Invocation
metadata omits reasoning effort when it is inherited, records the replacement
value when one is supplied, and records an empty value when inherited reasoning
was explicitly cleared.
Notarius never creates debug state without an explicit request and never Notarius never creates debug state without an explicit request and never
automatically deletes a requested bundle. If allocation succeeds, the command automatically deletes a requested bundle. If allocation succeeds, the command
@@ -255,14 +313,29 @@ Provider execution settings and the generation timeout come from the selected
PromptKit profile. The invocation-only **--reasoning-effort** and PromptKit profile. The invocation-only **--reasoning-effort** and
**--clear-reasoning-effort** controls may replace or clear that profile setting **--clear-reasoning-effort** controls may replace or clear that profile setting
for all LLM-backed calls in one run without changing the profile. PromptKit for all LLM-backed calls in one run without changing the profile. PromptKit
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the structural output repair happens within one structured-completion call. Its
complete module operation and validation chain as defined by effective `structured_output_repair_attempts` limit is resolved from the
[module bindings](config.md#module-bindings-and-validators). selected binding, then the pipeline, then the prompt declaration; see
[module bindings](config.md#module-bindings-and-validators). This is distinct
from Notarius binding **retries**, which rerun the complete module operation
and validation chain and do not consume or replenish the structural-repair
limit. The maintained production prompts declare one repair attempt, paid only
after a structural failure. One structured completion with repair budget **R**
makes at most **R + 1** serial provider calls. If one stage attempt performs
**C** structured completions, a binding with **retries: N** has a maximum of
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
corresponding invocation counts and budgets. This is an upper bound, not a
promise that every call reaches a provider.
Timeouts are layered. Caller cancellation is the outer authority. A positive Timeouts are layered. Caller cancellation is the outer authority. A positive
effective generation timeout adds an inner request deadline, while zero effective generation timeout adds an inner request deadline, while zero
disables only that generation deadline. The HTTP client timeout remains a disables only that generation deadline. The HTTP client timeout remains a
transport-wide cap. Notarius does not add another timeout around PromptKit. transport-wide cap. Notarius does not add another timeout around PromptKit.
Repairs are serial within the same caller context, so their worst-case latency
and cost follow the provider-call bound above; provision run deadlines and
provider budgets accordingly. Credentials remain optional unless the selected
PromptKit profile requires one, in which case preparation fails before a
provider call when its configured credential is unavailable.
The pinned upstream boundary and profile-format links are in The pinned upstream boundary and profile-format links are in
[PromptKit Integration](integrations/pkg-promptkit.md). [PromptKit Integration](integrations/pkg-promptkit.md).
@@ -279,6 +352,11 @@ positive value makes the effective active local-generation bound the smaller
of **total_llm** and that local limit, so a local limit of four permits no more of **total_llm** and that local limit, so a local limit of four permits no more
than four active local generations. than four active local generations.
The Notarius scheduler admits one logical structured completion and holds that
permit while PromptKit performs its serial corrective calls. PromptKit applies
its selected-backend admission to each provider call; Notarius does not
reacquire a permit or add another scheduler for a repair.
For a positive local limit, PromptKit owns its default waiting capacity and For a positive local limit, PromptKit owns its default waiting capacity and
admission behavior. When a PromptKit backend has admitted all active and queued admission behavior. When a PromptKit backend has admitted all active and queued
work, a new call fails as capacity exhaustion before generation. The adapter work, a new call fails as capacity exhaustion before generation. The adapter

View File

@@ -167,18 +167,29 @@ starting, waits for started work, and prevents output encoding.
## Validation ## Validation
Validation is a framework-managed boundary around outputs from chunk, extract, Validation is a framework-managed boundary around outputs from chunk, extract,
merge, and normalize stages. Validators receive immutable stage output merge, and normalize stages. Validators receive immutable stage output and
and make an explicit whole-output decision: approve, approve with warnings, or make an explicit whole-output decision: approve, approve with warnings,
reject. reject, fail, or skip when a runtime prerequisite is unavailable.
Typed artifact validators receive the domain value directly. Chunk validators Typed artifact validators receive the domain value directly. Chunk validators
receive source-zone chunks, while serialized validators receive immutable receive source-zone chunks, while serialized validators receive immutable
representation bytes and declared schema metadata. A validator registered for representation bytes and declared schema metadata. A validator registered for
one target or artifact kind cannot satisfy an incompatible selection. one target or artifact kind cannot satisfy an incompatible selection.
Rejection is a recorded pipeline outcome, not a framework execution error. The framework runs every applicable validator sequentially in configured order.
Validator execution failures are framework errors. Rejected output does not It aggregates rejections, exhausted validator failures, and skips before the
advance to the next stage. producer policy chooses a disposition. A completed rejection never advances.
With no rejection, an exhausted validator failure may fail the run or, under
the configured `warn_continue` policy, advance a structurally valid candidate
with explicit incomplete-validation provenance. Validators report findings;
they do not choose candidate disposition.
A completed rejection supplies a stable reason code for internal provenance
and bounded actionable correction guidance for the candidate producer. Reason
codes, validator keys, and operator-facing messages remain diagnostic data;
they are not model instructions. The framework constructs model-facing retry
text only from the semantic guidance and fails the contract rather than
inventing or truncating missing guidance.
Default validator chains are production composition policy and are registered Default validator chains are production composition policy and are registered
centrally by stage and module. Configuration may replace a stage-local default, centrally by stage and module. Configuration may replace a stage-local default,
@@ -195,6 +206,19 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
and interpretation of structured output. Provider adapters do not own source- and interpretation of structured output. Provider adapters do not own source-
or domain-specific prompt logic. or domain-specific prompt logic.
PromptKit owns bounded structural correction within one structured completion.
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
the two budgets must remain separate.
An LLM-backed producer can participate in semantic correction only when it
declares `single_response_v1` and returns the exact one response that directly
controlled its candidate. On an actionable rejection, the framework rebuilds
the ordinary request and appends only the latest defective response as an
`assistant` message plus one aggregated `user` correction message. This is a
fresh replacement request, not a growing conversation. The retry budgets,
terminal policy, and sensitive-data rationale are recorded in
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md).
When a model selects an application entity, callers must supply a contextual When a model selects an application entity, callers must supply a contextual
selection and deterministically attach the opaque application identity whenever selection and deterministically attach the opaque application identity whenever
the selection resolves exactly. Models do not receive or reproduce opaque the selection resolves exactly. Models do not receive or reproduce opaque
@@ -229,7 +253,8 @@ invalid or incompatible.
Run manifests record enough resolved pipeline, module, source, reference, and Run manifests record enough resolved pipeline, module, source, reference, and
LLM provenance to make a run auditable after configuration changes. Manifests LLM provenance to make a run auditable after configuration changes. Manifests
record identities and summaries rather than secret or large payload content. record identities and bounded validation summaries rather than secret, raw
model, correction, or large payload content.
## State, Output, And Safety ## State, Output, And Safety
@@ -249,18 +274,39 @@ an invocation that explicitly requests resume. Debug is never a cache input and
is never created without an explicit request. Pipeline modules receive is never created without an explicit request. Pipeline modules receive
collaborator interfaces and never physical roots. collaborator interfaces and never physical roots.
Only accepted, completely validated producer output is reusable checkpoint or
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
results cannot become cache or checkpoint inputs, even when a
`warn_continue` result is allowed to advance in the current run. This
ineligibility follows derived merge and normalize results and generated
references for the remainder of the run: current-run handoff remains allowed,
but no dependent cache or checkpoint may be loaded or published.
Writes are atomic where practical. Paths for writes, moves, overwrites, and Writes are atomic where practical. Paths for writes, moves, overwrites, and
deletion must be narrow and explicit. Notarius never automatically deletes deletion must be narrow and explicit. Notarius never automatically deletes
output or requested debug bundles; cache cleanup is explicit and recoverable. output or requested debug bundles; cache cleanup is explicit and recoverable.
Secrets must not appear in errors, logs, output, cache, debug summaries, Secrets must not appear in errors, logs, output, cache, debug summaries,
traces, manifests, documentation, examples, or redacted configuration. Debug traces, manifests, documentation, examples, or redacted configuration. Raw
assistant responses and complete correction messages are attempt-local and are
excluded from ordinary durable records and summaries; the requested detailed
debug trace is the sole diagnostic surface allowed to retain them. Debug
collection is allowlisted to application-owned payloads and must not capture collection is allowlisted to application-owned payloads and must not capture
unrelated process environment values or filesystem content. Trace data may unrelated process environment values or filesystem content. Trace data may
contain application data and therefore inherits its sensitivity; operators own contain application data and therefore inherits its sensitivity; operators own
access controls and retention. Physical layout and operation are defined in access controls and retention. Physical layout and operation are defined in
[Operations](../operations.md). [Operations](../operations.md).
## Platform And Distribution
Linux is the supported deployment platform. macOS is supported only as a
best-effort development and compilation environment, while Windows is
unsupported. Notarius distributes source releases only: an immutable source
tag and its checked-in release note identify a release. The project does not
publish executable binaries, archives, installers, container images,
checksums, signatures, or package-manager entries. Maintainer release commands
and tag guards belong to [Source Releases](../release.md).
## Architectural Non-Goals ## Architectural Non-Goals
Notarius does not aim to provide: Notarius does not aim to provide:

View File

@@ -65,6 +65,8 @@ secret values.
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. | | CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. | | Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. | | Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. | | Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. | | Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. | | External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
crosses these topics, choose the document that owns the task and link to the crosses these topics, choose the document that owns the task and link to the
other contracts. other contracts.
### Releases
`docs/release.md` owns the source-release procedure. Release notes are
historical summaries, not current-state contract owners: the checked-in note at
an immutable tag records that release, while current canonical documentation
must change with the behavior it describes. Do not use a release note to defer
or replace current documentation updates.
### Contracts And Implementation ### Contracts And Implementation
Integration and API documents define externally observable shapes and Integration and API documents define externally observable shapes and

164
docs/release.md Normal file
View File

@@ -0,0 +1,164 @@
# Source Releases
This procedure is for maintainers publishing Notarius source releases. A
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
candidate after publication; it does not publish or repair a release.
Notarius publishes no binaries, archives, checksums, signatures, containers,
package-manager entries, or Gitea release objects. Windows is not supported.
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
`v0.3.0` tags.
## Select And Describe The Release
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
documented CLI, configuration, durable artifact, integration, or operating
contract when its note explains the impact and required operator action. A
patch release must not intentionally break those documented contracts within
its minor line.
Create the version-matched note as part of the candidate. Every new note uses
this structure, with concise, truthful content in each section:
```markdown
# Notarius vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
The note is a historical summary. Link to current canonical documentation for
exact behavior, and update that documentation in the candidate rather than
using the note as a substitute.
## Prepare The Candidate
Set the selected release version and disable Go workspace use for every
candidate command:
```sh
RELEASE_VERSION=vMAJOR.MINOR.PATCH
export RELEASE_VERSION GOWORK=off
```
Run the shared source-candidate checks from the repository. They cover module
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
configuration validation, and the Linux and Darwin command-build matrix:
```sh
./scripts/check-release-source.sh "$RELEASE_VERSION"
```
Before committing, manually follow every changed local Markdown link and
review the candidate for unintended files, generated output, credentials, or
other unrelated changes. Commit the release note and all affected current
documentation, then run the shared checker against that exact candidate. Push
the candidate commit to `main` only after it succeeds. Record the exact commit
only after that push:
```sh
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
export RELEASE_COMMIT
```
For private-module installation, configure standard `GOPRIVATE` matching this
module and ordinary Git authentication for the hosting service before running
the verification below. The exact authentication mechanism belongs to the
maintainer environment; never record credentials or environment dumps in a
release note, command history, or repository file.
## Guard And Publish The Tag
Fetch current remote references, then run this guard without editing the
candidate. It requires `main`, a clean worktree and index, disabled workspace
use, a stable release version, the recorded and pushed commit, a matching note,
and unused local and remote tags:
```sh
git fetch origin main --tags
if ! printf '%s\n' "$RELEASE_VERSION" |
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
then
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
exit 1
fi
test "$GOWORK" = off
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
test -s "docs/releases/$RELEASE_VERSION.md"
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
done
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
```
Create an explicitly lightweight tag against the guarded commit, verify its
target, and push only that tag ref:
```sh
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
```
Never use `git push --tags`, move a published tag, or delete a published tag.
## Verify The Published Release
Confirm that the remote tag still points at the guarded commit and that the
note is available from the tagged tree:
```sh
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
```
Verify a fresh source installation and its diagnostic version. The temporary
directory confines the installed command to this check:
```sh
release_verification_dir=$(mktemp -d)
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
mkdir -p "$release_verification_dir/bin"
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
```
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
equivalent source verification when local installation policy requires it.
`notarius --version` is diagnostic only; downstream compatibility remains
defined by the published receipt and artifact contracts.
## Failure And Correction Policy
If candidate validation fails before publication, fix the candidate on `main`,
rerun the shared checker, and repeat the guards. An unpublished local tag may
be deleted after inspection.
If the remote tag or tag CI reveals a defect, leave the published tag intact.
Fix the defect on `main`, choose a new patch version, write a new matching
note, and repeat this procedure. Do not weaken tag immutability or add release
assets as a workaround.

83
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,83 @@
# Notarius v0.4.0
This release strengthens LLM reliability and validation throughout the
configured pipeline, upgrades the PromptKit integration, and establishes the
source-release and downstream-consumer workflows needed for broader D&D
pipeline integration.
## Summary
Notarius now distinguishes PromptKit structural-output repair from
application-owned semantic validation retries. Producer candidates can run
through complete deterministic validator chains, receive bounded semantic
correction guidance, and retry under explicit stage policies. Final run
receipts and manifests preserve bounded validation provenance, while outputs
that advance with incomplete validation remain available to the current run
without entering reusable checkpoint state.
The release also adds a maintained complete D&D subprocess-consumer workflow,
diagnostic build versions, and the source-only release procedure used to
publish this version.
## Compatibility
- Configuration files must use schema version 4. Version 3 is not decoded or
rewritten; rename the top-level `scriptorium` section to `promptkit` when
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
format and may use its profile-inheritance support. Notarius continues to
resolve operator profiles before embedded fallbacks.
- Structural-output repair and semantic stage retries are separate bounded
mechanisms. Maintained production prompts request one structural repair by
default; explicit configuration can override the supported repair count.
- Validation policy can now fail a run, reject an output, or permit an
otherwise valid candidate to advance with incomplete-validation provenance.
The application defaults are documented in
[Configuration](../config.md#pipelines).
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
optional validation summaries plus a required validation-status field.
Consumers of this pre-release contract should follow the current
[run-result receipt](../integrations/run-result.md).
- Existing D&D artifact schema identities remain unchanged. Validation and
producer-policy changes can nevertheless cause previously accepted weak
candidates to retry, reject, or fail instead.
## Upgrade
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
to `promptkit`.
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
and ensure their credential environment variables are available at run
time.
3. Run `notarius config validate --config <path> --pipeline <id>` before the
first production invocation.
4. Review `structured_output_repair_attempts`, producer retry counts, and
`validation_policy` wherever the deployment needs behavior different from
the documented defaults.
5. Update subprocess consumers to inspect receipt `validation_status` and to
tolerate the optional bounded `validation_summaries` field. A consumer that
requires fully validated artifacts should require `approved`.
## Changes
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
inheritance, structured-output repair, typed error classification, and the
correction-aware completion protocol.
- Added pipeline and binding configuration for structural repair and terminal
validation policy, with strict startup validation and effective-setting
provenance.
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
semantic reconciliation producers. Retry prompts contain the exact defective
response and actionable semantic correction guidance without exposing
internal reason codes or opaque entity identifiers.
- Added complete validator-chain execution, validator retry handling, bounded
warnings, terminal dispositions, and durable validation summaries.
- Prevented validation-incomplete artifacts and all derived lineage from
loading or publishing reusable checkpoints while preserving same-run
generated-reference handoff.
- Tightened cached chunk-plan validation so only completely validated plans are
reused or replace stored plans.
- Added a machine-readable subprocess receipt workflow and complete D&D
consumer documentation covering all maintained artifacts.
- Added source-release checks, immutable lightweight-tag guidance, Linux and
Darwin build verification, and diagnostic `notarius --version` output.

View File

@@ -5,13 +5,90 @@ configuration, operations, internal, and integration docs. This roadmap records
future work only. Items are ordered roughly by current value and specificity, future work only. Items are ordered roughly by current value and specificity,
not as committed release dates. not as committed release dates.
## Near-Term Validation And LLM Reliability
PromptKit now owns structural output repair within one completion. Notarius
owns stage candidates, validator chains, semantic rejection policy, bounded
feedback-aware stage retries, validation provenance, and reusable-state
eligibility. The remaining near-term work applies those completed foundations
to domain review and operator-facing diagnostics.
### D&D Combat Scene Semantic Validation
- Add an optional production LLM-backed D&D validator that determines whether
proposed scene boundaries and classifications represent substantive active
combat correctly. Its central quality goal is that active combat is kept in
coherent scenes classified as `combat`, rather than split incorrectly or
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
- Resolve the validator's exact target before implementation. The current
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
`recap`, and `meta` classification. The preferred initial placement is
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
can compare one proposed kind with the corresponding transcript chunk.
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
question that can be answered from the complete transcript and proposed
range map, such as whether one continuous combat was fragmented across
inappropriate scene boundaries. Do not duplicate the same classification
judgment at both stages. Moving classification into chunk-plan annotations
would change the deliberately minimal, annotation-free chunk contract and
requires an explicit architecture review before it is selected.
- Validate both false negatives and false positives: a non-combat kind must not
omit substantive active combat, and a combat kind must be supported by such
combat. Keep the existing deterministic downstream rule that combat-turn
extraction runs only for an exact `combat` scene classification; semantic
review improves the upstream classification but does not replace that gate.
- Run the semantic validator through PromptKit, use a minimal required-field
structured response schema, and let PromptKit repair structural validator
output within its bounded budget. A contract-invalid final validator response
is a validator execution failure, not a semantic rejection and not a reason
to recursively validate the validator.
- Evaluate the prompt and decision policy against a small human-reviewed set
containing combat setup, active turns, interruptions, multi-phase encounters,
brief rules discussion, aftermath, recalled combat, and false-positive
hostile dialogue. Measure false acceptance, false rejection, retry success,
added calls, latency, and token cost before placing it in the production
default chain.
- An ADR is not required if classification remains owned by
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
Create or supersede an ADR if the work transfers scene classification into
the chunker or otherwise changes stage ownership or the durable chunk-plan
contract.
### Warning Signal And Presentation Reform
- Audit every warning producer and representative successful runs. Ordinary
success producing dozens of warnings is a failed operator experience: the
volume obscures actionable problems and trains operators to ignore the
warning channel.
- Define a small warning taxonomy that distinguishes actionable degradation,
incomplete validation, lossy fallback, and data-quality risk from routine
normalization observations or informational diagnostics. Preserve detailed
traceability in debug or manifest data without promoting every observation
to a top-level CLI warning.
- Consider stable deduplication and aggregation by scope and reason code,
bounded samples plus omitted counts, and a concise CLI summary with a path to
detailed diagnostics. Do not suppress genuine validator execution failures
merely to reduce the count.
- Decide which warnings affect process status, rejection summaries, durable run
receipts, or only debug output. Ensure warning ordering and aggregation are
deterministic across concurrent execution.
- Establish a representative warning-volume acceptance target and human review
workflow before changing individual producers piecemeal. The intended result
is not zero warnings; it is a small set in which every surfaced warning merits
operator attention.
- This work does not require an ADR unless it changes validation acceptance,
failure, or durable contract semantics. CLI presentation and diagnostic
taxonomy otherwise belong in a feature roadmap followed by updates to their
canonical configuration, operations, integration, and internal documents.
## Near-Term D&D Pipeline ## Near-Term D&D Pipeline
### Evaluate Spell Extraction And Normalization ### Evaluate Spell Extraction And Normalization
- Evaluate ordinary extraction retries and the completed normalization path - Evaluate ordinary extraction retries and the completed normalization path
against a human-reviewed transcript set before adding repair-aware retries or against a human-reviewed transcript set before and after adopting the shared
an LLM-backed semantic validator. PromptKit repair and Notarius validation-retry policies above.
- Maintain a small set of human-reviewed transcripts and outputs for prompt, - Maintain a small set of human-reviewed transcripts and outputs for prompt,
validator, and normalizer development. Treat model-quality review as an validator, and normalizer development. Treat model-quality review as an
iterative human evaluation aid, not a deterministic correctness gate. iterative human evaluation aid, not a deterministic correctness gate.
@@ -28,8 +105,7 @@ The implemented source-backed core and initial D&D registry adoption are
described by [Module Internals](../internal/modules.md#semantic-reconciliation) described by [Module Internals](../internal/modules.md#semantic-reconciliation)
and and
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation). [D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
The [Semantic Reconciliation Roadmap](semantic-reconciliation.md) retains the The sections below keep broader extensions deferred.
original feature scope; the sections below keep broader extensions deferred.
### Large-Collection Semantic Reconciliation ### Large-Collection Semantic Reconciliation
@@ -153,7 +229,6 @@ section only after a concrete workflow, contract, and priority emerge.
### Distribution And Operations ### Distribution And Operations
- Packaged release artifacts for alpha distribution. - Packaged release artifacts for alpha distribution.
- A documented versioning and release process.
- Optional generated example-output fixtures with a regeneration procedure. - Optional generated example-output fixtures with a regeneration procedure.
- Additional diagnostics or reporting views. - Additional diagnostics or reporting views.

7
go.mod
View File

@@ -3,9 +3,14 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5 go 1.25.5
require ( require (
gitea.maximumdirect.net/eric/promptkit v0.5.0 gitea.maximumdirect.net/eric/promptkit v0.9.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
require golang.org/x/text v0.40.0 require golang.org/x/text v0.40.0
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
)

8
go.sum
View File

@@ -1,5 +1,9 @@
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc= gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ= gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

View File

@@ -0,0 +1,36 @@
// Package buildinfo resolves the product version embedded in a Notarius build.
package buildinfo
import (
"fmt"
"regexp"
"runtime/debug"
)
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
// Override is set at link time for controlled builds.
var Override string
// Version returns the release version embedded in the build, or development
// when the build does not carry a stable release tag.
func Version() (string, error) {
buildVersion := ""
if info, ok := debug.ReadBuildInfo(); ok {
buildVersion = info.Main.Version
}
return resolve(Override, buildVersion)
}
func resolve(override, buildVersion string) (string, error) {
if override != "" {
if !stableVersion.MatchString(override) {
return "", fmt.Errorf("build version override is not a stable release tag")
}
return override, nil
}
if stableVersion.MatchString(buildVersion) {
return buildVersion, nil
}
return "development", nil
}

View File

@@ -0,0 +1,45 @@
package buildinfo
import "testing"
func TestResolve(t *testing.T) {
tests := []struct {
name string
override string
buildVersion string
want string
wantErr bool
}{
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
{name: "development build", buildVersion: "(devel)", want: "development"},
{name: "missing build information", want: "development"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := resolve(tt.override, tt.buildVersion)
if tt.wantErr {
if err == nil {
t.Fatal("resolve() error = nil, want error")
}
return
}
if err != nil {
t.Fatalf("resolve() error = %v", err)
}
if got != tt.want {
t.Fatalf("resolve() = %q, want %q", got, tt.want)
}
})
}
}

View File

@@ -21,6 +21,10 @@ import (
const assembledSpellExtractorKey = "test/dnd/spell-casts" const assembledSpellExtractorKey = "test/dnd/spell-casts"
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{}) registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
@@ -101,6 +105,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
} }
} }
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
}
correction := extractor.correctionSnapshot()
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
}
}
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) { func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true}) registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
var normalizeChain *pipeline.ResolvedValidatorChain var normalizeChain *pipeline.ResolvedValidatorChain
@@ -136,6 +164,7 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) { func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true}) registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil { if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err) t.Fatalf("Prepare() error = %v, want nil", err)
@@ -161,8 +190,8 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) { if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected) t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
} }
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" { if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" || output.Warnings[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want terminal normalize catalog warning", output.Warnings) t.Fatalf("warnings = %#v, want complete terminal normalize validation warnings", output.Warnings)
} }
} }
@@ -206,6 +235,47 @@ type assembledSpellPipelineOptions struct {
unknownSpell bool unknownSpell bool
} }
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
extractor := &assembledCorrectingSpellExtractor{}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledCorrectingSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register correcting extractor: %v", err)
}
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
return assembledDirectSpellValidator{}, nil
}); err != nil {
t.Fatalf("register direct spell validator: %v", err)
}
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
extract.Retries = 1
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
},
Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return components.registries, resolved, extractor
}
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) { func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
t.Helper() t.Helper()
components := productionTestComponents(t) components := productionTestComponents(t)
@@ -251,6 +321,69 @@ type assembledSpellExtractor struct {
unknownSpell bool unknownSpell bool
} }
type assembledCorrectingSpellExtractor struct {
mu sync.Mutex
correction *contracts.SemanticCorrection
}
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if req.Source == nil || req.Chunk == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
}
response := `{"spell":"accepted"}`
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
if req.Chunk.Index == 0 && req.Correction == nil {
response = `{"spell":"Mysterious Burst"}`
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
if req.Chunk.Index == 0 && req.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
e.mu.Lock()
e.correction = correction
e.mu.Unlock()
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
}
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
e.mu.Lock()
defer e.mu.Unlock()
correction, err := contracts.CloneSemanticCorrection(e.correction)
if err != nil {
return nil
}
return correction
}
type assembledDirectSpellValidator struct{}
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
for _, cast := range req.Value.SpellCasts {
if cast.Spell == "Mysterious Burst" {
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
}
}
return contracts.ValidationResult{Approved: true}, nil
}
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey } func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }

View File

@@ -8,6 +8,8 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
) )
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) { func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"}, {name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"}, {name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"}, {name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
} }
} }
func TestCommandVersionOutput(t *testing.T) {
previous := buildinfo.Override
t.Cleanup(func() { buildinfo.Override = previous })
tests := []struct {
name string
override string
wantCode int
wantStdout string
wantStderr string
}{
{name: "development", wantStdout: "notarius development\n"},
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buildinfo.Override = tt.override
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) { func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
explicit := writeCommandConfig(t, "explicit", "alpha") explicit := writeCommandConfig(t, "explicit", "alpha")
environment := writeCommandConfig(t, "environment", "beta") environment := writeCommandConfig(t, "environment", "beta")

View File

@@ -189,10 +189,18 @@ func TestMaintainedCompleteExamplePublishesRegistryBackedEntityOccurrences(t *te
} }
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json")) evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
for _, laneID := range []string{"enemy-events", "npc-registry", "npc-occurrences", "item-registry", "item-occurrences", "location-registry", "location-occurrences"} { if len(evidence) == 0 {
if !containsString(evidence.SelectedLanes, laneID) || !evidenceHasLane(evidence, laneID) { t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
t.Fatalf("evidence context = %#v, want direct %s evidence", evidence, laneID)
} }
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
for _, unit := range evidence {
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
}
if _, exists := seenEvidenceUnits[unit.ID]; exists {
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
}
seenEvidenceUnits[unit.ID] = struct{}{}
} }
requests := client.requestsFor(enemyevents.PromptID) requests := client.requestsFor(enemyevents.PromptID)
@@ -388,8 +396,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
if err := json.Unmarshal(content, output); err != nil { if err := json.Unmarshal(content, output); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
} }
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, request) client.requests = append(client.requests, snapshot)
client.mu.Unlock() client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
} }
@@ -400,7 +412,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
var requests []contracts.StructuredCompletionRequest var requests []contracts.StructuredCompletionRequest
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if request.PromptID == promptID {
requests = append(requests, request) snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
} }
} }
return requests return requests
@@ -415,17 +431,6 @@ func containsString(values []string, want string) bool {
return false return false
} }
func evidenceHasLane(value evidencecontext.Document, laneID string) bool {
for _, context := range value.Contexts {
for _, reference := range context.EvidenceRefs {
if reference.LaneID == laneID {
return true
}
}
}
return false
}
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) { func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
for _, binding := range bindings { for _, binding := range bindings {
if binding.SlotName == slotName && binding.Artifact != nil { if binding.SlotName == slotName && binding.Artifact != nil {

View File

@@ -19,6 +19,8 @@ import (
"testing/fstest" "testing/fstest"
"time" "time"
"gopkg.in/yaml.v3"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
@@ -331,27 +333,61 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("production prompt engine: %v", err) t.Fatalf("production prompt engine: %v", err)
} }
inputs := map[string]promptkit.ArtifactRef{ promptFS, err := components.assets.PromptFS()
"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Alias","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), if err != nil {
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`), t.Fatalf("production prompt assets: %v", err)
}
type manifest struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
Inputs []struct {
Name string `yaml:"name"`
} `yaml:"inputs"`
Messages []struct {
Role string `yaml:"role"`
} `yaml:"messages"`
}
preparedPrompts := 0
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
return nil
}
data, err := fs.ReadFile(promptFS, path)
if err != nil {
return err
}
var prompt manifest
if err := yaml.Unmarshal(data, &prompt); err != nil {
return err
}
for _, message := range prompt.Messages {
if message.Role != promptkit.RoleSystem && message.Role != promptkit.RoleUser {
return fmt.Errorf("production prompt %q uses role %q, want system or user", prompt.ID, message.Role)
}
}
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
for _, input := range prompt.Inputs {
inputs[input.Name] = promptkit.Inline(`{}`)
} }
for _, prompt := range []struct {
id string
version string
}{
{id: npcnormalize.PromptID, version: npcnormalize.PromptVersion},
{id: itemregistrynormalize.PromptID, version: itemregistrynormalize.PromptVersion},
{id: locationnormalize.PromptID, version: locationnormalize.PromptVersion},
} {
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: prompt.id, PromptVersion: prompt.version, ProfileID: "assembled-prompt-test", Inputs: inputs, PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
}) })
if err != nil { if err != nil {
t.Fatalf("prepare production prompt %q: %v", prompt.id, err) return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
} }
if prepared.OutputContract.SchemaPath != filepath.Base(semanticreconcile.SchemaAssetPath) { if prepared.OutputContract.RepairAttempts != 1 {
t.Fatalf("prompt %q schema = %q, want generic reconciliation schema", prompt.id, prepared.OutputContract.SchemaPath) return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
} }
preparedPrompts++
return nil
}); err != nil {
t.Fatal(err)
}
if preparedPrompts == 0 {
t.Fatal("prepared no production prompts")
} }
} }
@@ -1063,8 +1099,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
if err := json.Unmarshal(content, out); err != nil { if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
} }
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, req) client.requests = append(client.requests, snapshot)
client.mu.Unlock() client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
} }
@@ -1075,7 +1115,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
var requests []contracts.StructuredCompletionRequest var requests []contracts.StructuredCompletionRequest
for _, req := range client.requests { for _, req := range client.requests {
if req.PromptID == promptID { if req.PromptID == promptID {
requests = append(requests, req) snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
} }
} }
return requests return requests

View File

@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
}, },
{ {
name: "malformed profile", name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"), profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
profileID: "malformed-profile", profileID: "malformed-profile",
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`}, wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
rejectErr: []string{"malformed-profile.yaml", "backend: ["}, rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
}, },
{ {
name: "invalid profile source", name: "invalid profile source",
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
} }
} }
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
for _, profiles := range []string{
"id: child\nbase_profile: missing\n",
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
} {
t.Run("invalid inheritance", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
t.Fatal(err)
}
profileID := "child"
if strings.Contains(profiles, "id: first") {
profileID = "first"
}
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
t.Fatalf("profile preflight error = %v", err)
}
})
}
}

View File

@@ -16,6 +16,7 @@ import (
"strings" "strings"
"time" "time"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
@@ -31,6 +32,7 @@ import (
const defaultConfigPath = "/usr/local/etc/notarius/config.yml" const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
const usage = `Usage: const usage = `Usage:
notarius help notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags] notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius pipelines list --config path/to/config.yml [--json]
@@ -62,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
} }
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int { func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
if len(args) > 0 && args[0] == "--version" {
if len(args) != 1 {
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
return 2
}
version, err := buildinfo.Version()
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "notarius %s\n", version)
return 0
}
var err error var err error
opts, err = normalizeOptions(opts) opts, err = normalizeOptions(opts)
if err != nil { if err != nil {

View File

@@ -48,8 +48,15 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if info.Mode().Perm() != want { mode := info.Mode().Perm()
t.Fatalf("%s mode = %#o, want %#o", path, info.Mode().Perm(), want) if mode&^want != 0 {
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
}
if info.IsDir() && mode&0o700 != 0o700 {
t.Fatalf("%s mode = %#o, want owner access", path, mode)
}
if !info.IsDir() && mode != want {
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
} }
} }
entries, err := os.ReadDir(filepath.Join(runPath, "nested")) entries, err := os.ReadDir(filepath.Join(runPath, "nested"))

View File

@@ -7,6 +7,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -22,6 +23,7 @@ type runResult struct {
RejectedOutputCount int `json:"rejected_output_count"` RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"` WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status"` ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"` DebugDirectory string `json:"debug_directory,omitempty"`
} }
@@ -59,6 +61,7 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
RejectedOutputCount: len(output.Rejected), RejectedOutputCount: len(output.Rejected),
WarningCount: len(output.Warnings), WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus, ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
} }
if strings.TrimSpace(debugDirectory) != "" { if strings.TrimSpace(debugDirectory) != "" {
@@ -85,6 +88,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
return result, nil return result, nil
} }
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
}
func encodeRunResult(result runResult) ([]byte, error) { func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result) encoded, err := json.Marshal(result)
if err != nil { if err != nil {

View File

@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n")) configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil { if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -55,6 +55,9 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
if got := decoded["warning_count"]; got != float64(1) { if got := decoded["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got) t.Fatalf("warning_count = %v", got)
} }
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
}
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") { if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got) t.Fatalf("output_directory = %q", got)
} }
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
} }
} }
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
output := testRunOutput()
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
if err != nil {
t.Fatal(err)
}
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
t.Fatalf("encoded result = %s", encoded)
}
}
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) { func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
t.Fatalf("materialize production references: %v", err) t.Fatalf("materialize production references: %v", err)
} }
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
llmClient := &catalogRetryLLMClient{responses: tt.responses} llmClient := &catalogRetryLLMClient{responses: tt.responses}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient}) prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 { if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection) t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
} }
if len(output.Warnings) != 0 { if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want no emitted warnings from rejected attempts", output.Warnings) t.Fatalf("warnings = %#v, want complete terminal validation warnings", output.Warnings)
} }
return return
} }

View File

@@ -104,6 +104,33 @@ type RejectedOutputManifest struct {
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
AttemptCount int `json:"attempt_count,omitempty"` AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Validation *ValidationSummary `json:"validation,omitempty"`
}
// ValidationSummary is the bounded, durable outcome of validating one
// producer result. It deliberately contains identities and stable codes, not
// model responses, corrective guidance, validator diagnostics, or payloads.
type ValidationSummary struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Status string `json:"status"`
RejectingValidators []string `json:"rejecting_validators,omitempty"`
ReasonCodes []string `json:"reason_codes,omitempty"`
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
ProducerAttemptCount int `json:"producer_attempt_count"`
TerminalAction string `json:"terminal_action"`
}
// CloneValidationSummary returns an independently owned durable summary.
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
return summary
} }
type CheckpointDecisionManifest struct { type CheckpointDecisionManifest struct {
@@ -163,6 +190,7 @@ type RunManifest struct {
References []ReferenceProvenance `json:"references,omitempty"` References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"` NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"` RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"` CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`

View File

@@ -116,6 +116,11 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile { func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
@@ -196,6 +201,11 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
if len(in.Options) > 0 { if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options) out.Options = cloneOptions(in.Options)
} }
@@ -204,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out return out
} }
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride { func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set} out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 { if len(in.Validators) > 0 {

View File

@@ -36,6 +36,8 @@ type FilePromptKitLocalBackendConfig struct {
type FilePipelineProfile struct { type FilePipelineProfile struct {
LLMProfile *string `yaml:"llm_profile,omitempty"` LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
Input fileModuleBinding `yaml:"input"` Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
@@ -48,15 +50,25 @@ type FilePipelineProfile struct {
} }
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error { func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
return err
}
type plainFilePipelineProfile FilePipelineProfile type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{ seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {}, "llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile") }, "pipeline profile")
if err != nil { if err != nil {
return err return err
} }
*p = FilePipelineProfile(decoded) *p = FilePipelineProfile(decoded)
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
if err != nil {
return err
}
p.ValidationPolicy = policy
}
_, p.artifactsSet = seen["artifacts"] _, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"] _, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"] _, p.llmProfileSet = seen["llm_profile"]
@@ -149,6 +161,8 @@ type FileDebugConfig struct {
type fileModuleBinding struct { type fileModuleBinding struct {
Module string Module string
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
ValidationPolicy *pipeline.ValidationPolicyOverride
Retries int Retries int
Options map[string]any Options map[string]any
References map[string]fileReferenceSource References map[string]fileReferenceSource
@@ -245,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
b.Module = strings.TrimSpace(module) b.Module = strings.TrimSpace(module)
return nil return nil
case yaml.MappingNode: case yaml.MappingNode:
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i] keyNode := node.Content[i]
valueNode := node.Content[i+1] valueNode := node.Content[i+1]
if _, exists := seen[keyNode.Value]; exists {
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
}
seen[keyNode.Value] = struct{}{}
switch keyNode.Value { switch keyNode.Value {
case "module": case "module":
var module string var module string
@@ -264,6 +283,18 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
if b.LLMProfile == "" { if b.LLMProfile == "" {
return fmt.Errorf("llm_profile must not be empty when set") return fmt.Errorf("llm_profile must not be empty when set")
} }
case "structured_output_repair_attempts":
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
if err != nil {
return err
}
b.StructuredOutputRepairAttempts = attempts
case "validation_policy":
policy, err := parseValidationPolicy(valueNode, "module binding")
if err != nil {
return err
}
b.ValidationPolicy = policy
case "retries": case "retries":
var retries int var retries int
if err := valueNode.Decode(&retries); err != nil { if err := valueNode.Decode(&retries); err != nil {
@@ -306,13 +337,100 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
return pipeline.ModuleBinding{ return pipeline.ModuleBinding{
Module: strings.TrimSpace(b.Module), Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile), LLMProfile: strings.TrimSpace(b.LLMProfile),
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
Retries: b.Retries, Retries: b.Retries,
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References), References: fileReferenceSourcesToPipeline(b.References),
Validators: b.Validators, Validators: cloneValidatorOverride(b.Validators),
} }
} }
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value == key {
return node.Content[i+1], true
}
}
return nil, false
}
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s validation_policy must be an object", context)
}
policy := &pipeline.ValidationPolicyOverride{}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
value := node.Content[i+1]
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
}
seen[key] = struct{}{}
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
}
switch key {
case "producer_structural_failure":
value := pipeline.ProducerStructuralFailureAction(value.Value)
policy.ProducerStructuralFailure = &value
case "semantic_rejection":
value := pipeline.SemanticRejectionAction(value.Value)
policy.SemanticRejection = &value
case "validator_failure":
value := pipeline.ValidatorFailureAction(value.Value)
policy.ValidatorFailure = &value
default:
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
}
}
if err := policy.Validate(); err != nil {
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
}
return policy, nil
}
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("%s must be an object", context)
}
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value != "structured_output_repair_attempts" {
continue
}
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
return err
}
}
return nil
}
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
if node.Tag == "!!null" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
}
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
}
var attempts int
if err := node.Decode(&attempts); err != nil {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
}
if attempts < 0 || attempts > 3 {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return &attempts, nil
}
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
if attempts == nil {
return nil
}
value := *attempts
return &value
}
func LoadFileConfig(path string) (FileConfig, error) { func LoadFileConfig(path string) (FileConfig, error) {
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -520,6 +638,8 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
profile := pipeline.PipelineProfile{ profile := pipeline.PipelineProfile{
ID: pipelineID, ID: pipelineID,
LLMProfile: llmProfile, LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
Input: filePipeline.Input.toPipelineBinding(), Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References), References: fileReferenceSourcesToPipeline(filePipeline.References),

View File

@@ -138,6 +138,168 @@ pipelines:
} }
} }
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
%s
input: input
artifacts:
lane:
extract:
module: extract
structured_output_repair_attempts: 2
validators:
- module: validator
structured_output_repair_attempts: 3
`
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
}
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
}
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
}
})
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
profile := cfg.Pipelines["main"]
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
}
lane := profile.Artifacts["lane"]
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
}
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
}
fileProfile := file.Pipelines["main"]
fileLane := fileProfile.Artifacts["lane"]
*fileLane.Extract.StructuredOutputRepairAttempts = 1
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
fileProfile.Artifacts["lane"] = fileLane
file.Pipelines["main"] = fileProfile
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
}
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
}
cloned := cloneConfig(cfg)
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
}
redacted := cfg.Redacted()
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
}
summary := cfg.RedactedSummaryPayload().(Config)
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatal(err)
}
var roundTripped Config
if err := json.Unmarshal(data, &roundTripped); err != nil {
t.Fatal(err)
}
roundTrippedProfile := roundTripped.Pipelines["main"]
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
}
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
}
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
})
}
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
input: input
artifacts:
lane:
extract: extract
%s
`
const bindingYAML = `version: 4
pipelines:
main:
input:
module: input
%s
artifacts:
lane:
extract: extract
`
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
}
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) { func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
const configYAML = `version: 4 const configYAML = `version: 4
pipelines: pipelines:

View File

@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline { func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
out := in out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = redactBinding(cloneModuleBinding(in.Input)) out.Input = redactBinding(cloneModuleBinding(in.Input))
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk)) out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences) out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)

View File

@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
} }
} }
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
semantic := pipeline.SemanticRejectionRejectOutput
validator := pipeline.ValidatorFailureFailRun
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
effective := EffectiveConfig{
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
"main": {ValidationPolicy: configured},
}},
ResolvedPipeline: pipeline.ResolvedPipeline{
ConfiguredValidationPolicy: configured,
ChunkValidationPolicy: pipeline.ValidationPolicy{
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
ValidatorFailure: pipeline.ValidatorFailureFailRun,
},
},
}
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
encoded, err := json.Marshal(payload)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
}
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatal("redacted payload aliases validation policy")
}
}
func redactionTestBinding(name string) pipeline.ModuleBinding { func redactionTestBinding(name string) pipeline.ModuleBinding {
return pipeline.ModuleBinding{ return pipeline.ModuleBinding{
Module: "safe-" + name, Module: "safe-" + name,

View File

@@ -116,6 +116,14 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if profile.ID != "" && strings.TrimSpace(profile.ID) != id { if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID) return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
} }
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
return err
}
if profile.ValidationPolicy != nil {
if err := profile.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
}
}
if err := validateBinding(id, "", "input", profile.Input, false); err != nil { if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err return err
} }
@@ -195,6 +203,19 @@ func validateBinding(
binding pipeline.ModuleBinding, binding pipeline.ModuleBinding,
referencesAllowed bool, referencesAllowed bool,
) error { ) error {
if binding.ValidationPolicy != nil {
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
}
if err := binding.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
}
}
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
return err
}
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil { if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
return err return err
} }
@@ -219,6 +240,13 @@ func validateBinding(
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true) return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
} }
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
if attempts != nil && (*attempts < 0 || *attempts > 3) {
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return nil
}
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error { func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set { if !override.Set {
return nil return nil
@@ -230,6 +258,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
} }
for i, validator := range override.Validators { for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i) context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
return err
}
if strings.TrimSpace(validator.Module) == "" { if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context) return fmt.Errorf("%s module must not be empty", context)
} }
@@ -239,8 +270,11 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
if validator.Validators.Set { if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context) return fmt.Errorf("%s nested validators are not supported", context)
} }
if validator.Retries != 0 { if validator.ValidationPolicy != nil {
return fmt.Errorf("%s retries are not supported", context) return fmt.Errorf("%s validation_policy is not supported", context)
}
if validator.Retries < 0 {
return fmt.Errorf("%s retries must be greater than or equal to zero", context)
} }
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" { if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context) return fmt.Errorf("%s llm_profile must not be empty when set", context)

View File

@@ -374,17 +374,17 @@ func TestValidateValidatorBindingRules(t *testing.T) {
want: "chunk validators[0] module must not be empty", want: "chunk validators[0] module must not be empty",
}, },
{ {
name: "validator retries", name: "negative validator retries",
setup: func(profile *pipeline.PipelineProfile) { setup: func(profile *pipeline.PipelineProfile) {
profile.Chunk.Validators = pipeline.ValidatorOverride{ profile.Chunk.Validators = pipeline.ValidatorOverride{
Set: true, Set: true,
Validators: []pipeline.ModuleBinding{{ Validators: []pipeline.ModuleBinding{{
Module: "validator", Module: "validator",
Retries: 1, Retries: -1,
}}, }},
} }
}, },
want: "chunk validators[0] retries are not supported", want: "chunk validators[0] retries must be greater than or equal to zero",
}, },
{ {
name: "validator references", name: "validator references",

View File

@@ -0,0 +1,97 @@
package config
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
const valid = `version: 4
pipelines:
main:
validation_policy:
producer_structural_failure: reject_output
semantic_rejection: fail_run
input: seriatim
chunk:
module: generic
validation_policy:
validator_failure: fail_run
artifacts:
lane:
extract:
module: extract
validation_policy:
semantic_rejection: reject_output
`
cfg := applyFileConfig(t, valid)
profile := cfg.Pipelines["main"]
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
}
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
}
lane := profile.Artifacts["lane"]
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
}
for _, test := range []struct {
name string
yaml string
}{
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
}
})
}
}
func TestValidationPolicyPlacementRules(t *testing.T) {
policy := &pipeline.ValidationPolicyOverride{}
semantic := pipeline.SemanticRejectionRejectOutput
policy.SemanticRejection = &semantic
base := pipeline.PipelineProfile{
ID: "main",
Input: pipeline.Binding("input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"lane": {Extract: pipeline.Binding("extract")},
},
}
for _, test := range []struct {
name string
mutate func(*pipeline.PipelineProfile)
}{
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
{"output", func(profile *pipeline.PipelineProfile) {
profile.Output = pipeline.Binding("output")
profile.Output.ValidationPolicy = policy
}},
{"validator", func(profile *pipeline.PipelineProfile) {
lane := profile.Artifacts["lane"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
profile.Artifacts["lane"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := base
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
test.mutate(&profile)
cfg := Default()
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
}
})
}
}

View File

@@ -10,6 +10,7 @@ import (
"strings" "strings"
"time" "time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -344,7 +345,15 @@ func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.Rejec
if len(rejected) == 0 { if len(rejected) == 0 {
return nil return nil
} }
return append([]contracts.RejectedOutput(nil), rejected...) cloned := make([]contracts.RejectedOutput, len(rejected))
for index, item := range rejected {
cloned[index] = item
if item.Validation != nil {
summary := artifacts.CloneValidationSummary(*item.Validation)
cloned[index].Validation = &summary
}
}
return cloned
} }
func cloneMetadata(metadata map[string]any) map[string]any { func cloneMetadata(metadata map[string]any) map[string]any {

View File

@@ -0,0 +1,84 @@
package contracts
import (
"crypto/sha256"
"encoding/hex"
"fmt"
)
// DebugStructuredCompletionRequest is the content-safe representation of a
// structured completion request for ordinary diagnostics and summaries.
// Detailed prompt material remains available only through the explicitly
// requested LLM debug trace.
type DebugStructuredCompletionRequest struct {
StageName string `json:"stage_name,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
InputCount int `json:"input_count"`
VariableCount int `json:"variable_count"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
Correction *DebugSemanticCorrection `json:"correction,omitempty"`
}
// DebugSemanticCorrection records only safe correction metadata. It never
// exposes the assistant response or user guidance text.
type DebugSemanticCorrection struct {
AssistantResponseBytes int `json:"assistant_response_bytes"`
AssistantResponseDigest string `json:"assistant_response_digest"`
UserGuidanceBytes int `json:"user_guidance_bytes"`
UserGuidanceDigest string `json:"user_guidance_digest"`
}
// DebugSummary returns a content-safe representation suitable for ordinary
// diagnostics. It does not validate or retain correction content.
func (request StructuredCompletionRequest) DebugSummary() DebugStructuredCompletionRequest {
summary := DebugStructuredCompletionRequest{
StageName: request.StageName,
PromptID: request.PromptID,
PromptVersion: request.PromptVersion,
ProfileID: request.ProfileID,
SessionID: request.SessionID,
InputCount: len(request.Inputs),
VariableCount: len(request.Vars),
}
if request.StructuredOutputRepairAttempts != nil {
attempts := *request.StructuredOutputRepairAttempts
summary.StructuredOutputRepairAttempts = &attempts
}
if request.Correction != nil {
summary.Correction = request.Correction.DebugSummary()
}
return summary
}
// DebugSummary returns content-safe correction metadata suitable for ordinary
// diagnostics.
func (correction *SemanticCorrection) DebugSummary() *DebugSemanticCorrection {
if correction == nil {
return nil
}
return &DebugSemanticCorrection{
AssistantResponseBytes: len(correction.AssistantResponse),
AssistantResponseDigest: debugContentDigest(correction.AssistantResponse),
UserGuidanceBytes: len(correction.UserGuidance),
UserGuidanceDigest: debugContentDigest([]byte(correction.UserGuidance)),
}
}
// String prevents ordinary request formatting from exposing correction
// content. Use the explicitly requested debug trace for complete messages.
func (request StructuredCompletionRequest) String() string {
return fmt.Sprintf("%+v", request.DebugSummary())
}
// GoString gives %#v formatting the same content-safe behavior as String.
func (request StructuredCompletionRequest) GoString() string {
return request.String()
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -16,6 +16,8 @@ type StructuredCompletionRequest struct {
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
Inputs LLMInputSet `json:"inputs,omitempty"` Inputs LLMInputSet `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"` Vars map[string]any `json:"vars,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
Correction *SemanticCorrection `json:"-"`
} }
type StructuredCompletionResponse struct { type StructuredCompletionResponse struct {
@@ -26,6 +28,7 @@ type StructuredCompletionResponse struct {
PromptTokens int `json:"prompt_tokens,omitempty"` PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"` CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"` TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"` Debug *LLMDebugMaterial `json:"debug,omitempty"`
} }
@@ -73,6 +76,14 @@ type LLMDebugResponse struct {
InputHashes map[string]string `json:"input_hashes,omitempty"` InputHashes map[string]string `json:"input_hashes,omitempty"`
Validation map[string]any `json:"validation,omitempty"` Validation map[string]any `json:"validation,omitempty"`
Usage LLMDebugUsage `json:"usage,omitempty"` Usage LLMDebugUsage `json:"usage,omitempty"`
ProviderError *LLMDebugProviderError `json:"provider_error,omitempty"`
}
type LLMDebugProviderError struct {
StatusCode int `json:"status_code,omitempty"`
Code string `json:"code,omitempty"`
Type string `json:"type,omitempty"`
Message string `json:"message,omitempty"`
} }
type LLMDebugUsage struct { type LLMDebugUsage struct {
@@ -130,6 +141,7 @@ type ParseRequest struct {
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
Raw []byte `json:"-"` Raw []byte `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
@@ -144,12 +156,15 @@ type ChunkRequest struct {
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"` References ReferenceSet `json:"references,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
Correction *SemanticCorrection `json:"-"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
type ChunkPlanResult struct { type ChunkPlanResult struct {
Plan source.ChunkPlan `json:"plan"` Plan source.ChunkPlan `json:"plan"`
Warnings []Warning `json:"warnings,omitempty"` Warnings []Warning `json:"warnings,omitempty"`
ModelCandidate *ModelCandidate `json:"-"`
} }
type Chunker interface { type Chunker interface {
@@ -271,6 +286,7 @@ type ValidationResult struct {
Approved bool `json:"approved"` Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"` ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
CorrectionGuidance string `json:"-"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Warnings []Warning `json:"warnings,omitempty"` Warnings []Warning `json:"warnings,omitempty"`
} }
@@ -287,6 +303,7 @@ type OutputRequest struct {
Rejected []RejectedOutput `json:"rejected,omitempty"` Rejected []RejectedOutput `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"` Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"` ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"` EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
@@ -320,6 +337,7 @@ type RejectedOutput struct {
Message string `json:"message"` Message string `json:"message"`
AttemptCount int `json:"attempt_count,omitempty"` AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Validation *artifacts.ValidationSummary `json:"validation,omitempty"`
} }
type ManifestMetadataProvider interface { type ManifestMetadataProvider interface {

View File

@@ -0,0 +1,175 @@
package contracts
import (
"errors"
"fmt"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
// CorrectionProtocol identifies how a producer can represent the model output
// that directly controlled a candidate.
type CorrectionProtocol string
const (
CorrectionProtocolSingleResponseV1 CorrectionProtocol = "single_response_v1"
)
const (
MaxValidationReasonCodeBytes = 128
MaxValidationCorrectionGuidanceBytes = 4 * 1024
MaxAssistantResponseBytes = 1 << 20
MaxCorrectionGuidanceBytes = 64 * 1024
MaxCorrectionContentBytes = MaxAssistantResponseBytes + MaxCorrectionGuidanceBytes
)
func (protocol CorrectionProtocol) Validate() error {
if protocol == CorrectionProtocolSingleResponseV1 {
return nil
}
return fmt.Errorf("unsupported correction protocol %q", protocol)
}
// SemanticCorrection carries the latest model response and application-owned
// guidance for one fresh corrected request. Its content is sensitive and is
// deliberately excluded from ordinary JSON serialization.
type SemanticCorrection struct {
AssistantResponse []byte `json:"-"`
UserGuidance string `json:"-"`
}
func NewSemanticCorrection(assistantResponse []byte, userGuidance string) (*SemanticCorrection, error) {
correction := &SemanticCorrection{
AssistantResponse: append([]byte(nil), assistantResponse...),
UserGuidance: userGuidance,
}
if err := correction.Validate(); err != nil {
return nil, err
}
return correction, nil
}
func CloneSemanticCorrection(correction *SemanticCorrection) (*SemanticCorrection, error) {
if correction == nil {
return nil, nil
}
return NewSemanticCorrection(correction.AssistantResponse, correction.UserGuidance)
}
// CloneStructuredCompletionRequest returns a request whose mutable values are
// owned by the caller. It is suitable for clients that retain requests after
// CompleteStructured returns.
func CloneStructuredCompletionRequest(request StructuredCompletionRequest) (StructuredCompletionRequest, error) {
correction, err := CloneSemanticCorrection(request.Correction)
if err != nil {
return StructuredCompletionRequest{}, fmt.Errorf("clone correction: %w", err)
}
vars, err := source.CloneMetadata(request.Vars)
if err != nil {
return StructuredCompletionRequest{}, fmt.Errorf("clone variables: %w", err)
}
request.Inputs = request.Inputs.Clone()
request.Vars = vars
request.Correction = correction
if request.StructuredOutputRepairAttempts != nil {
attempts := *request.StructuredOutputRepairAttempts
request.StructuredOutputRepairAttempts = &attempts
}
return request, nil
}
func (correction SemanticCorrection) Validate() error {
if err := validateAssistantResponse(correction.AssistantResponse); err != nil {
return fmt.Errorf("semantic correction assistant response: %w", err)
}
if err := validateBoundedText(correction.UserGuidance, MaxCorrectionGuidanceBytes, "semantic correction user guidance", false); err != nil {
return err
}
if len(correction.AssistantResponse)+len(correction.UserGuidance) > MaxCorrectionContentBytes {
return errors.New("semantic correction content exceeds maximum length")
}
return nil
}
// ModelCandidate preserves the exact single model response that directly
// controlled a producer result. It is attempt-local and never durable data.
type ModelCandidate struct {
Response []byte `json:"-"`
Protocol CorrectionProtocol `json:"-"`
}
func NewModelCandidate(response []byte, protocol CorrectionProtocol) (*ModelCandidate, error) {
candidate := &ModelCandidate{Response: append([]byte(nil), response...), Protocol: protocol}
if err := candidate.Validate(); err != nil {
return nil, err
}
return candidate, nil
}
func CloneModelCandidate(candidate *ModelCandidate) (*ModelCandidate, error) {
if candidate == nil {
return nil, nil
}
return NewModelCandidate(candidate.Response, candidate.Protocol)
}
func (candidate ModelCandidate) Validate() error {
if err := candidate.Protocol.Validate(); err != nil {
return err
}
if err := validateAssistantResponse(candidate.Response); err != nil {
return fmt.Errorf("model candidate response: %w", err)
}
return nil
}
func ValidateValidationResult(result ValidationResult) error {
if !result.Approved && result.ReasonCode == "" {
return errors.New("validation rejection reason code must not be empty")
}
if result.ReasonCode != "" {
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
return err
}
}
if !result.Approved && result.CorrectionGuidance == "" {
return errors.New("validation rejection correction guidance must not be empty")
}
if result.CorrectionGuidance != "" {
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
return err
}
}
return nil
}
func validateAssistantResponse(response []byte) error {
if len(response) > MaxAssistantResponseBytes {
return errors.New("exceeds maximum length")
}
if !utf8.Valid(response) {
return errors.New("must be valid UTF-8")
}
if len(strings.TrimSpace(string(response))) == 0 {
return errors.New("must not be blank")
}
return nil
}
func validateBoundedText(value string, maximum int, name string, optional bool) error {
if value == "" && optional {
return nil
}
if !utf8.ValidString(value) {
return fmt.Errorf("%s must be valid UTF-8", name)
}
if strings.TrimSpace(value) == "" {
return fmt.Errorf("%s must not be blank", name)
}
if len(value) > maximum {
return fmt.Errorf("%s exceeds maximum length", name)
}
return nil
}

View File

@@ -0,0 +1,184 @@
package contracts
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"testing"
)
func TestSemanticCorrectionOwnsValidatedContent(t *testing.T) {
assistant := []byte(`{"items":["original"]}`)
correction, err := NewSemanticCorrection(assistant, "Return one corrected replacement.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
assistant[0] = '['
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
t.Fatalf("assistant response = %q, want owned original content", got)
}
clone, err := CloneSemanticCorrection(correction)
if err != nil {
t.Fatalf("CloneSemanticCorrection() error = %v", err)
}
clone.AssistantResponse[0] = '['
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
t.Fatalf("source correction changed through clone = %q", got)
}
if nilClone, err := CloneSemanticCorrection(nil); err != nil || nilClone != nil {
t.Fatalf("CloneSemanticCorrection(nil) = %#v, %v; want nil, nil", nilClone, err)
}
encoded, err := json.Marshal(correction)
if err != nil {
t.Fatalf("marshal correction: %v", err)
}
if string(encoded) != "{}" {
t.Fatalf("correction JSON = %s, want no sensitive content", encoded)
}
}
func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
tooLongAssistant := bytes.Repeat([]byte("a"), MaxAssistantResponseBytes+1)
tooLongGuidance := strings.Repeat("a", MaxCorrectionGuidanceBytes+1)
tooLongReason := strings.Repeat("a", MaxValidationReasonCodeBytes+1)
tooLongValidationGuidance := strings.Repeat("a", MaxValidationCorrectionGuidanceBytes+1)
for _, test := range []struct {
name string
call func() error
}{
{"blank assistant", func() error { _, err := NewSemanticCorrection([]byte(" \n"), "guidance"); return err }},
{"invalid assistant utf8", func() error { _, err := NewSemanticCorrection([]byte{0xff}, "guidance"); return err }},
{"oversized assistant", func() error { _, err := NewSemanticCorrection(tooLongAssistant, "guidance"); return err }},
{"blank guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), " \t"); return err }},
{"invalid guidance utf8", func() error { _, err := NewSemanticCorrection([]byte("response"), string([]byte{0xff})); return err }},
{"oversized guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), tooLongGuidance); return err }},
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
{"missing rejection reason code", func() error {
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
}},
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
{"oversized reason code", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
}},
{"blank reason code", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
}},
{"invalid correction guidance utf8", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
}},
{"oversized correction guidance", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
}},
} {
t.Run(test.name, func(t *testing.T) {
if err := test.call(); err == nil {
t.Fatal("validation error = nil, want error")
}
})
}
}
func TestModelCandidateOwnsValidatedResponse(t *testing.T) {
response := []byte(`{"items":["original"]}`)
candidate, err := NewModelCandidate(response, CorrectionProtocolSingleResponseV1)
if err != nil {
t.Fatalf("NewModelCandidate() error = %v", err)
}
response[0] = '['
if got := string(candidate.Response); got != `{"items":["original"]}` {
t.Fatalf("candidate response = %q, want owned original content", got)
}
clone, err := CloneModelCandidate(candidate)
if err != nil {
t.Fatalf("CloneModelCandidate() error = %v", err)
}
clone.Response[0] = '['
if got := string(candidate.Response); got != `{"items":["original"]}` {
t.Fatalf("source candidate changed through clone = %q", got)
}
if nilClone, err := CloneModelCandidate(nil); err != nil || nilClone != nil {
t.Fatalf("CloneModelCandidate(nil) = %#v, %v; want nil, nil", nilClone, err)
}
}
func TestValidationResultAllowsAbsentOptionalCorrectionFields(t *testing.T) {
if err := ValidateValidationResult(ValidationResult{Approved: true}); err != nil {
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
}
if err := ValidateValidationResult(ValidationResult{ReasonCode: "invalid-evidence", CorrectionGuidance: "Provide source-backed evidence."}); err != nil {
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
}
if err := CorrectionProtocol("").Validate(); err == nil {
t.Fatal("empty correction protocol validation error = nil, want error")
}
}
func TestCloneStructuredCompletionRequestOwnsCorrection(t *testing.T) {
correction, err := NewSemanticCorrection([]byte(`{"value":"original"}`), "Correct the value.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
attempts := 2
request := StructuredCompletionRequest{
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
Vars: map[string]any{"labels": []string{"original"}},
StructuredOutputRepairAttempts: &attempts,
Correction: correction,
}
clone, err := CloneStructuredCompletionRequest(request)
if err != nil {
t.Fatalf("CloneStructuredCompletionRequest() error = %v", err)
}
correction.AssistantResponse[0] = '['
request.Inputs["source"] = NewLLMInputMaterial("source", "application/json", []byte(`{"source":false}`), "", "")
*request.StructuredOutputRepairAttempts = 7
if got := string(clone.Correction.AssistantResponse); got != `{"value":"original"}` {
t.Fatalf("cloned correction response = %q, want owned original content", got)
}
if got := string(clone.Inputs["source"].Content); got != `{"source":true}` {
t.Fatalf("cloned input = %q, want owned original content", got)
}
if clone.StructuredOutputRepairAttempts == nil || *clone.StructuredOutputRepairAttempts != 2 {
t.Fatalf("cloned repair attempts = %v, want 2", clone.StructuredOutputRepairAttempts)
}
}
func TestStructuredCompletionRequestDebugSummaryOmitsCorrectionContent(t *testing.T) {
const assistantResponse = `{"secret":"assistant response"}`
const userGuidance = "secret user guidance"
correction, err := NewSemanticCorrection([]byte(assistantResponse), userGuidance)
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
request := StructuredCompletionRequest{
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
Vars: map[string]any{"custom": "value"},
Correction: correction,
}
summary := request.DebugSummary()
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
t.Fatalf("debug summary = %#v, want input, variable, and correction metadata", summary)
}
if summary.Correction.AssistantResponseBytes != len(assistantResponse) || summary.Correction.UserGuidanceBytes != len(userGuidance) ||
summary.Correction.AssistantResponseDigest == "" || summary.Correction.UserGuidanceDigest == "" {
t.Fatalf("correction summary = %#v, want byte counts and digests", summary.Correction)
}
encoded, err := json.Marshal(summary)
if err != nil {
t.Fatalf("marshal debug summary: %v", err)
}
for _, rendered := range []string{string(encoded), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request)} {
for _, secret := range []string{assistantResponse, userGuidance} {
if strings.Contains(rendered, secret) {
t.Fatalf("content-safe request rendering leaked %q: %s", secret, rendered)
}
}
}
}

View File

@@ -9,3 +9,31 @@ var ErrInvalidStructuredOutput = errors.New("invalid structured output")
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model // ErrLLMCapacityExceeded identifies backend admission exhaustion before model
// generation begins. // generation begins.
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded") var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
// ErrLLMGeneration identifies a provider generation failure.
var ErrLLMGeneration = errors.New("LLM generation failed")
type LLMGenerationError struct {
status int
diagnostic string
}
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
if status < 0 {
status = 0
}
return &LLMGenerationError{status: status, diagnostic: diagnostic}
}
func (e *LLMGenerationError) Error() string {
if e == nil || e.diagnostic == "" {
return ErrLLMGeneration.Error()
}
return e.diagnostic
}
func (e *LLMGenerationError) Unwrap() error { return ErrLLMGeneration }
func (e *LLMGenerationError) StatusCode() int {
if e == nil {
return 0
}
return e.status
}

View File

@@ -41,12 +41,15 @@ type TypedExtractionRequest struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Correction *SemanticCorrection
Metadata map[string]any Metadata map[string]any
} }
type TypedExtractionResult[T any] struct { type TypedExtractionResult[T any] struct {
Value T Value T
Warnings []Warning Warnings []Warning
ModelCandidate *ModelCandidate
} }
type Extractor[T any] interface { type Extractor[T any] interface {
@@ -63,12 +66,15 @@ type TypedMergeRequest[T any] struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Correction *SemanticCorrection
Metadata map[string]any Metadata map[string]any
} }
type TypedMergeResult[T any] struct { type TypedMergeResult[T any] struct {
Value T Value T
Warnings []Warning Warnings []Warning
ModelCandidate *ModelCandidate
} }
type Merger[T any] interface { type Merger[T any] interface {
@@ -84,6 +90,8 @@ type TypedNormalizeRequest[T any] struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Correction *SemanticCorrection
Metadata map[string]any Metadata map[string]any
} }
@@ -91,6 +99,7 @@ type TypedNormalizeResult[T any] struct {
Value T Value T
Warnings []Warning Warnings []Warning
Retry *NormalizeRetry Retry *NormalizeRetry
ModelCandidate *ModelCandidate
} }
// Normalize retry diagnostic limits bound module-provided values before the // Normalize retry diagnostic limits bound module-provided values before the
@@ -124,6 +133,7 @@ type TypedValidationRequest[T any] struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Metadata map[string]any Metadata map[string]any
Chunk *source.Chunk Chunk *source.Chunk
Chunks []source.Chunk Chunks []source.Chunk
@@ -145,6 +155,7 @@ type ChunkValidationRequest struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Metadata map[string]any Metadata map[string]any
Chunks []source.Chunk Chunks []source.Chunk
} }
@@ -165,6 +176,7 @@ type SerializedValidationRequest struct {
SessionID string SessionID string
References ReferenceSet References ReferenceSet
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int
Metadata map[string]any Metadata map[string]any
Chunk *source.Chunk Chunk *source.Chunk
Chunks []source.Chunk Chunks []source.Chunk

View File

@@ -2,24 +2,8 @@
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.source.evidence_context", "$id": "notarius.source.evidence_context",
"title": "notarius_source_evidence_context_v1", "title": "notarius_source_evidence_context_v1",
"type": "object",
"additionalProperties": false,
"required": ["source_id", "source_digest", "window_units", "selected_lanes", "contexts"],
"properties": {
"source_id": {"type": "string", "minLength": 1},
"source_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
"window_units": {"type": "integer", "minimum": 0},
"selected_lanes": {
"type": "array", "type": "array",
"minItems": 1, "items": {"$ref": "#/$defs/unit"},
"uniqueItems": true,
"items": {"type": "string", "minLength": 1}
},
"contexts": {
"type": "array",
"items": {"$ref": "#/$defs/context"}
}
},
"$defs": { "$defs": {
"source_ref": { "source_ref": {
"type": "object", "type": "object",
@@ -42,25 +26,6 @@
"ref": {"$ref": "#/$defs/source_ref"}, "ref": {"$ref": "#/$defs/source_ref"},
"metadata": {"type": "object", "additionalProperties": true} "metadata": {"type": "object", "additionalProperties": true}
} }
},
"evidence_ref": {
"type": "object",
"additionalProperties": false,
"required": ["lane_id", "source_ref"],
"properties": {
"lane_id": {"type": "string", "minLength": 1},
"source_ref": {"$ref": "#/$defs/source_ref"}
}
},
"context": {
"type": "object",
"additionalProperties": false,
"required": ["context_ref", "evidence_refs", "units"],
"properties": {
"context_ref": {"$ref": "#/$defs/source_ref"},
"evidence_refs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evidence_ref"}},
"units": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/unit"}}
}
} }
} }
} }

View File

@@ -3,118 +3,62 @@ package evidencecontext
import ( import (
"fmt" "fmt"
"sort" "sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
) )
type contribution struct {
laneID string
ref source.SourceRef
startPos int
endPos int
}
type expandedRange struct { type expandedRange struct {
startPos int startPos int
endPos int endPos int
contributions []contribution
} }
// Build validates accepted direct references, expands them by source-document // Build validates projected source references, expands them by source-document
// position, and returns their deterministic context union. // position, and returns their ordered union as an owned source-unit excerpt.
func Build(request BuildRequest) (Document, error) { func Build(request BuildRequest) (Document, error) {
if request.WindowUnits < 0 { if request.WindowUnits < 0 {
return Document{}, fmt.Errorf("window_units must not be negative") return nil, fmt.Errorf("window_units must not be negative")
}
lanes, err := normalizeSelectedLanes(request.SelectedLanes)
if err != nil {
return Document{}, err
} }
if err := source.ValidateDocument(request.Source); err != nil { if err := source.ValidateDocument(request.Source); err != nil {
return Document{}, fmt.Errorf("validate source document: %w", err) return nil, fmt.Errorf("validate source document: %w", err)
} }
digest, err := source.DigestDocument(request.Source) digest, err := source.DigestDocument(request.Source)
if err != nil { if err != nil {
return Document{}, fmt.Errorf("digest source document: %w", err) return nil, fmt.Errorf("digest source document: %w", err)
} }
if digest != request.Source.Digest { if digest != request.Source.Digest {
return Document{}, fmt.Errorf("source digest does not match source document digest") return nil, fmt.Errorf("source digest does not match source document digest")
} }
selected := make(map[string]struct{}, len(lanes))
for _, laneID := range lanes {
selected[laneID] = struct{}{}
}
index := source.NewDocumentIndex(request.Source) index := source.NewDocumentIndex(request.Source)
seen := make(map[evidenceKey]struct{}) ranges := make([]expandedRange, 0, len(request.SourceRefs))
contributions := make([]contribution, 0) for refIndex, ref := range request.SourceRefs {
for laneIndex, laneEvidence := range request.LaneEvidence {
laneID := strings.TrimSpace(laneEvidence.LaneID)
if _, ok := selected[laneID]; !ok {
return Document{}, fmt.Errorf("lane evidence[%d] lane %q is not selected", laneIndex, laneID)
}
for refIndex, ref := range laneEvidence.SourceRefs {
if err := index.ValidateRef(ref); err != nil { if err := index.ValidateRef(ref); err != nil {
return Document{}, fmt.Errorf("lane %q source reference[%d]: %w", laneID, refIndex, err) return nil, fmt.Errorf("source reference[%d]: %w", refIndex, err)
} }
key := evidenceKey{laneID: laneID, ref: ref}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
startPos, _ := index.Position(ref.StartUnitID) startPos, _ := index.Position(ref.StartUnitID)
endPos, _ := index.Position(ref.EndUnitID) endPos, _ := index.Position(ref.EndUnitID)
contributions = append(contributions, contribution{laneID: laneID, ref: ref, startPos: expandStart(startPos, request.WindowUnits), endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits)}) ranges = append(ranges, expandedRange{
} startPos: expandStart(startPos, request.WindowUnits),
endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits),
})
} }
sort.Slice(contributions, func(i, j int) bool { return lessContribution(contributions[i], contributions[j]) }) merged := mergeRanges(ranges)
document := Document{ unitCount := 0
SourceID: request.Source.ID, for _, value := range merged {
SourceDigest: digest, unitCount += value.endPos - value.startPos + 1
WindowUnits: request.WindowUnits,
SelectedLanes: lanes,
Contexts: make([]Context, 0),
} }
for _, rangeValue := range mergeRanges(contributions) { document := make(Document, 0, unitCount)
context, err := buildContext(request.Source, rangeValue) for _, value := range merged {
for position := value.startPos; position <= value.endPos; position++ {
unit, err := cloneSourceUnit(request.Source.Units[position])
if err != nil { if err != nil {
return Document{}, err return nil, fmt.Errorf("clone source unit at position %d: %w", position, err)
} }
document.Contexts = append(document.Contexts, context) document = append(document, unit)
} }
canonical, err := canonicalizeOwned(document)
if err != nil {
return Document{}, fmt.Errorf("validate evidence context: %w", err)
} }
return canonical, nil return document, nil
}
type evidenceKey struct {
laneID string
ref source.SourceRef
}
func normalizeSelectedLanes(values []string) ([]string, error) {
if len(values) == 0 {
return nil, fmt.Errorf("selected_lanes must not be empty")
}
seen := make(map[string]struct{}, len(values))
lanes := make([]string, 0, len(values))
for index, raw := range values {
laneID := strings.TrimSpace(raw)
if laneID == "" {
return nil, fmt.Errorf("selected_lanes[%d] must not be empty", index)
}
if _, exists := seen[laneID]; exists {
return nil, fmt.Errorf("selected_lanes lane %q is duplicated", laneID)
}
seen[laneID] = struct{}{}
lanes = append(lanes, laneID)
}
sort.Strings(lanes)
return lanes, nil
} }
func expandStart(position, window int) int { func expandStart(position, window int) int {
@@ -132,65 +76,25 @@ func expandEnd(position, length, window int) int {
return position + window return position + window
} }
func lessContribution(left, right contribution) bool { func mergeRanges(values []expandedRange) []expandedRange {
if left.startPos != right.startPos {
return left.startPos < right.startPos
}
if left.endPos != right.endPos {
return left.endPos < right.endPos
}
return lessEvidenceRef(EvidenceRef{LaneID: left.laneID, SourceRef: left.ref}, EvidenceRef{LaneID: right.laneID, SourceRef: right.ref})
}
func mergeRanges(values []contribution) []expandedRange {
if len(values) == 0 { if len(values) == 0 {
return nil return nil
} }
ranges := make([]expandedRange, 0, len(values)) sort.Slice(values, func(i, j int) bool {
if values[i].startPos != values[j].startPos {
return values[i].startPos < values[j].startPos
}
return values[i].endPos < values[j].endPos
})
merged := make([]expandedRange, 0, len(values))
for _, value := range values { for _, value := range values {
if len(ranges) == 0 || value.startPos > ranges[len(ranges)-1].endPos+1 { if len(merged) == 0 || value.startPos > merged[len(merged)-1].endPos+1 {
ranges = append(ranges, expandedRange{startPos: value.startPos, endPos: value.endPos, contributions: []contribution{value}}) merged = append(merged, value)
continue continue
} }
current := &ranges[len(ranges)-1] if value.endPos > merged[len(merged)-1].endPos {
if value.endPos > current.endPos { merged[len(merged)-1].endPos = value.endPos
current.endPos = value.endPos
} }
current.contributions = append(current.contributions, value)
} }
return ranges return merged
}
func buildContext(document *source.SourceDocument, value expandedRange) (Context, error) {
evidenceRefs := make([]EvidenceRef, 0, len(value.contributions))
for _, contribution := range value.contributions {
evidenceRefs = append(evidenceRefs, EvidenceRef{LaneID: contribution.laneID, SourceRef: contribution.ref})
}
sort.Slice(evidenceRefs, func(i, j int) bool { return lessEvidenceRef(evidenceRefs[i], evidenceRefs[j]) })
units := make([]source.SourceUnit, 0, value.endPos-value.startPos+1)
for position := value.startPos; position <= value.endPos; position++ {
unit, err := cloneSourceUnit(document.Units[position])
if err != nil {
return Context{}, fmt.Errorf("clone source unit at position %d: %w", position, err)
}
units = append(units, unit)
}
return Context{
ContextRef: source.SourceRef{SourceID: document.ID, StartUnitID: units[0].ID, EndUnitID: units[len(units)-1].ID},
EvidenceRefs: evidenceRefs,
Units: units,
}, nil
}
func lessEvidenceRef(left, right EvidenceRef) bool {
if left.LaneID != right.LaneID {
return left.LaneID < right.LaneID
}
if left.SourceRef.SourceID != right.SourceRef.SourceID {
return left.SourceRef.SourceID < right.SourceRef.SourceID
}
if left.SourceRef.StartUnitID != right.SourceRef.StartUnitID {
return left.SourceRef.StartUnitID < right.SourceRef.StartUnitID
}
return left.SourceRef.EndUnitID < right.SourceRef.EndUnitID
} }

View File

@@ -6,7 +6,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"regexp"
"strings" "strings"
"sync" "sync"
@@ -18,8 +17,6 @@ import (
//go:embed assets/schemas/source_evidence_context.v1.json //go:embed assets/schemas/source_evidence_context.v1.json
var schemaAssets embed.FS var schemaAssets embed.FS
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
var ( var (
loadSchemaOnce sync.Once loadSchemaOnce sync.Once
loadedSchema []byte loadedSchema []byte
@@ -78,24 +75,24 @@ func (c *Codec) Encode(value Document) ([]byte, error) {
func (c *Codec) Decode(content []byte) (Document, error) { func (c *Codec) Decode(content []byte) (Document, error) {
if _, err := c.schemaBytes(); err != nil { if _, err := c.schemaBytes(); err != nil {
return Document{}, err return nil, err
} }
if err := validateSchemaInstance(content); err != nil { if err := validateSchemaInstance(content); err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err) return nil, fmt.Errorf("decode evidence context: %w", err)
} }
decoder := json.NewDecoder(bytes.NewReader(content)) decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields() decoder.DisallowUnknownFields()
var value Document var value Document
if err := decoder.Decode(&value); err != nil { if err := decoder.Decode(&value); err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err) return nil, fmt.Errorf("decode evidence context: %w", err)
} }
var trailing any var trailing any
if err := decoder.Decode(&trailing); err != io.EOF { if err := decoder.Decode(&trailing); err != io.EOF {
return Document{}, fmt.Errorf("decode evidence context: multiple JSON values") return nil, fmt.Errorf("decode evidence context: multiple JSON values")
} }
canonical, err := canonicalizeOwned(value) canonical, err := canonicalizeOwned(value)
if err != nil { if err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err) return nil, fmt.Errorf("decode evidence context: %w", err)
} }
return canonical, nil return canonical, nil
} }
@@ -118,14 +115,13 @@ func loadAndCompileSchema() {
ID string `json:"$id"` ID string `json:"$id"`
Title string `json:"title"` Title string `json:"title"`
Type string `json:"type"` Type string `json:"type"`
Required []string `json:"required"`
} }
if err := json.Unmarshal(raw, &identity); err != nil { if err := json.Unmarshal(raw, &identity); err != nil {
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err) loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
return return
} }
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) { if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "array" {
loadSchemaErr = fmt.Errorf("source evidence context schema identity or required fields are invalid") loadSchemaErr = fmt.Errorf("source evidence context schema identity is invalid")
return return
} }
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw)) schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
@@ -158,164 +154,65 @@ func validateSchemaInstance(content []byte) error {
return nil return nil
} }
func hasRequiredFields(required []string) bool {
want := map[string]bool{"source_id": true, "source_digest": true, "window_units": true, "selected_lanes": true, "contexts": true}
for _, field := range required {
delete(want, field)
}
return len(want) == 0
}
func canonicalize(value Document) (Document, error) { func canonicalize(value Document) (Document, error) {
owned, err := clone(value) owned, err := clone(value)
if err != nil { if err != nil {
return Document{}, err return nil, err
} }
return canonicalizeOwned(owned) return canonicalizeOwned(owned)
} }
func canonicalizeOwned(value Document) (Document, error) { func canonicalizeOwned(value Document) (Document, error) {
if err := requireIdentity("source_id", value.SourceID); err != nil { if value == nil {
return Document{}, err return nil, fmt.Errorf("document must be a JSON array")
} }
if !digestPattern.MatchString(value.SourceDigest) { seenUnitIDs := make(map[int]struct{}, len(value))
return Document{}, fmt.Errorf("source_digest must be a sha256 digest") sourceID := ""
for unitIndex := range value {
unit := value[unitIndex]
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
return nil, fmt.Errorf("units[%d] has invalid required fields", unitIndex)
} }
if value.WindowUnits < 0 { if err := validateUnitRef(unit, unitIndex); err != nil {
return Document{}, fmt.Errorf("window_units must not be negative") return nil, err
} }
if err := validateSelectedLanes(value.SelectedLanes); err != nil { if sourceID == "" {
return Document{}, err sourceID = unit.Ref.SourceID
} else if unit.Ref.SourceID != sourceID {
return nil, fmt.Errorf("units[%d].ref.source_id must match units[0].ref.source_id", unitIndex)
} }
if value.Contexts == nil { if _, exists := seenUnitIDs[unit.ID]; exists {
value.Contexts = make([]Context, 0) return nil, fmt.Errorf("units contains duplicate unit id %d", unit.ID)
} }
selected := make(map[string]struct{}, len(value.SelectedLanes)) seenUnitIDs[unit.ID] = struct{}{}
for _, laneID := range value.SelectedLanes {
selected[laneID] = struct{}{}
}
seenUnits := make(map[int]struct{})
for contextIndex := range value.Contexts {
context, err := canonicalizeContext(value.SourceID, selected, seenUnits, value.Contexts[contextIndex], contextIndex)
if err != nil {
return Document{}, err
}
value.Contexts[contextIndex] = context
} }
return value, nil return value, nil
} }
func validateSelectedLanes(lanes []string) error { func validateUnitRef(unit source.SourceUnit, unitIndex int) error {
if len(lanes) == 0 { prefix := fmt.Sprintf("units[%d].ref", unitIndex)
return fmt.Errorf("selected_lanes must not be empty") if strings.TrimSpace(unit.Ref.SourceID) == "" || strings.TrimSpace(unit.Ref.SourceID) != unit.Ref.SourceID {
} return fmt.Errorf("%s.source_id must be a non-empty trimmed string", prefix)
for index, laneID := range lanes {
if err := requireIdentity(fmt.Sprintf("selected_lanes[%d]", index), laneID); err != nil {
return err
}
if index > 0 && lanes[index-1] >= laneID {
return fmt.Errorf("selected_lanes must be unique and in lexical order")
}
}
return nil
}
func canonicalizeContext(sourceID string, selected map[string]struct{}, seenUnits map[int]struct{}, value Context, contextIndex int) (Context, error) {
prefix := fmt.Sprintf("contexts[%d]", contextIndex)
if len(value.EvidenceRefs) == 0 {
return Context{}, fmt.Errorf("%s.evidence_refs must not be empty", prefix)
}
if len(value.Units) == 0 {
return Context{}, fmt.Errorf("%s.units must not be empty", prefix)
}
if err := validateRefIdentity(sourceID, value.ContextRef, prefix+".context_ref"); err != nil {
return Context{}, err
}
positions := make(map[int]int, len(value.Units))
for unitIndex := range value.Units {
unit := value.Units[unitIndex]
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
return Context{}, fmt.Errorf("%s.units[%d] has invalid required fields", prefix, unitIndex)
}
if err := validateRefIdentity(sourceID, unit.Ref, fmt.Sprintf("%s.units[%d].ref", prefix, unitIndex)); err != nil {
return Context{}, err
} }
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID { if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
return Context{}, fmt.Errorf("%s.units[%d].ref must identify unit id %d", prefix, unitIndex, unit.ID) return fmt.Errorf("%s must identify unit id %d", prefix, unit.ID)
}
if _, exists := positions[unit.ID]; exists {
return Context{}, fmt.Errorf("%s.units contains duplicate unit id %d", prefix, unit.ID)
}
if _, exists := seenUnits[unit.ID]; exists {
return Context{}, fmt.Errorf("contexts contain duplicate unit id %d", unit.ID)
}
positions[unit.ID] = unitIndex
seenUnits[unit.ID] = struct{}{}
}
if value.ContextRef.StartUnitID != value.Units[0].ID || value.ContextRef.EndUnitID != value.Units[len(value.Units)-1].ID {
return Context{}, fmt.Errorf("%s.context_ref must identify the first and last units", prefix)
}
for evidenceIndex := range value.EvidenceRefs {
evidence := value.EvidenceRefs[evidenceIndex]
if _, ok := selected[evidence.LaneID]; !ok {
return Context{}, fmt.Errorf("%s.evidence_refs[%d].lane_id is not selected", prefix, evidenceIndex)
}
if err := requireIdentity(fmt.Sprintf("%s.evidence_refs[%d].lane_id", prefix, evidenceIndex), evidence.LaneID); err != nil {
return Context{}, err
}
if err := validateRefIdentity(sourceID, evidence.SourceRef, fmt.Sprintf("%s.evidence_refs[%d].source_ref", prefix, evidenceIndex)); err != nil {
return Context{}, err
}
start, startOK := positions[evidence.SourceRef.StartUnitID]
end, endOK := positions[evidence.SourceRef.EndUnitID]
if !startOK || !endOK || start > end {
return Context{}, fmt.Errorf("%s.evidence_refs[%d].source_ref is outside context units", prefix, evidenceIndex)
}
if evidenceIndex > 0 && !lessEvidenceRef(value.EvidenceRefs[evidenceIndex-1], evidence) {
return Context{}, fmt.Errorf("%s.evidence_refs must be unique and in deterministic order", prefix)
}
}
return value, nil
}
func validateRefIdentity(sourceID string, ref source.SourceRef, field string) error {
if ref.SourceID != sourceID {
return fmt.Errorf("%s.source_id does not match source_id", field)
}
if ref.StartUnitID <= 0 || ref.EndUnitID <= 0 {
return fmt.Errorf("%s endpoints must be positive", field)
}
return nil
}
func requireIdentity(field, value string) error {
if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value {
return fmt.Errorf("%s must be a non-empty trimmed string", field)
} }
return nil return nil
} }
func clone(value Document) (Document, error) { func clone(value Document) (Document, error) {
value.SelectedLanes = append([]string(nil), value.SelectedLanes...) if value == nil {
if value.Contexts == nil { return nil, nil
value.Contexts = make([]Context, 0) }
} else { cloned := make(Document, len(value))
contexts := make([]Context, len(value.Contexts)) for unitIndex, unit := range value {
for contextIndex, context := range value.Contexts { owned, err := cloneSourceUnit(unit)
contexts[contextIndex].ContextRef = context.ContextRef
contexts[contextIndex].EvidenceRefs = append([]EvidenceRef(nil), context.EvidenceRefs...)
contexts[contextIndex].Units = make([]source.SourceUnit, len(context.Units))
for unitIndex, unit := range context.Units {
cloned, err := cloneSourceUnit(unit)
if err != nil { if err != nil {
return Document{}, fmt.Errorf("clone contexts[%d].units[%d]: %w", contextIndex, unitIndex, err) return nil, fmt.Errorf("clone units[%d]: %w", unitIndex, err)
} }
contexts[contextIndex].Units[unitIndex] = cloned cloned[unitIndex] = owned
} }
} return cloned, nil
value.Contexts = contexts
}
return value, nil
} }
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) { func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {

View File

@@ -2,7 +2,6 @@ package evidencecontext
import ( import (
"bytes" "bytes"
"encoding/json"
"math" "math"
"os" "os"
"reflect" "reflect"
@@ -12,126 +11,57 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
) )
func TestBuildExpandsAndMergesEvidenceByDocumentPosition(t *testing.T) { func TestBuildSelectsExpandedSourceUnitUnion(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name string name string
window int window int
evidence []LaneEvidence refs []source.SourceRef
wantUnits [][]int wantIDs []int
wantRefs [][]EvidenceRef
}{ }{
{ {name: "zero window", refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{3}},
name: "zero window", {name: "multi unit citation includes complete range", refs: []source.SourceRef{ref(3, 7)}, wantIDs: []int{3, 30, 7}},
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}, {name: "non monotonic IDs use document positions", window: 1, refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{10, 3, 30}},
wantUnits: [][]int{{3}}, {name: "boundary clamping", window: 1, refs: []source.SourceRef{ref(10, 10), ref(50, 50)}, wantIDs: []int{10, 3, 7, 50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}}, {name: "overlapping and adjacent windows merge", window: 1, refs: []source.SourceRef{ref(3, 3), ref(30, 30), ref(30, 30)}, wantIDs: []int{10, 3, 30, 7}},
}, {name: "adjacent expanded ranges merge", window: 1, refs: []source.SourceRef{ref(10, 10), ref(7, 7)}, wantIDs: []int{10, 3, 30, 7, 50}},
{ {name: "largest window clips without overflow", window: math.MaxInt, refs: []source.SourceRef{ref(30, 30)}, wantIDs: []int{10, 3, 30, 7, 50}},
name: "non monotonic ids use positions and clip boundaries", {name: "no references returns an initialized empty document", wantIDs: []int{}},
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
wantUnits: [][]int{{10, 3, 30}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
},
{
name: "separate gaps stay separate",
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(50, 50)}}},
wantUnits: [][]int{{10}, {50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(10, 10)}}, {{LaneID: "npcs", SourceRef: ref(50, 50)}}},
},
{
name: "overlapping windows merge",
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}},
wantUnits: [][]int{{10, 3, 30, 7}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}, {LaneID: "npcs", SourceRef: ref(30, 30)}}},
},
{
name: "contiguous windows merge",
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(7, 7)}}},
wantUnits: [][]int{{10, 3, 30, 7, 50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(7, 7)}, {LaneID: "npcs", SourceRef: ref(10, 10)}}},
},
{
name: "duplicate contributions retain unique lane attribution",
evidence: []LaneEvidence{
{LaneID: "spells", SourceRefs: []source.SourceRef{ref(30, 30), ref(30, 30)}},
{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}},
},
wantUnits: [][]int{{30}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}, {LaneID: "spells", SourceRef: ref(30, 30)}}},
},
{
name: "empty contributions retain explicit empty contexts",
evidence: []LaneEvidence{{LaneID: "npcs"}},
wantUnits: [][]int{},
wantRefs: [][]EvidenceRef{},
},
{
name: "largest window clips without overflow",
window: math.MaxInt,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}},
wantUnits: [][]int{{10, 3, 30, 7, 50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}}},
},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
document := testDocument(t) got, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: test.window, SourceRefs: test.refs})
got, err := Build(BuildRequest{Source: document, WindowUnits: test.window, SelectedLanes: []string{"spells", "npcs"}, LaneEvidence: test.evidence})
if err != nil { if err != nil {
t.Fatalf("Build() error = %v", err) t.Fatalf("Build() error = %v", err)
} }
if want := []string{"npcs", "spells"}; !reflect.DeepEqual(got.SelectedLanes, want) { if got == nil {
t.Fatalf("SelectedLanes = %#v, want %#v", got.SelectedLanes, want) t.Fatal("Build() returned a nil document")
} }
if got.WindowUnits != test.window || got.SourceID != document.ID || got.SourceDigest != document.Digest { if actual := unitIDs(got); !reflect.DeepEqual(actual, test.wantIDs) {
t.Fatalf("Build() identity = %#v, want source and window identity", got) t.Fatalf("unit IDs = %#v, want %#v", actual, test.wantIDs)
}
if actual := contextUnitIDs(got.Contexts); !reflect.DeepEqual(actual, test.wantUnits) {
t.Fatalf("context unit ids = %#v, want %#v", actual, test.wantUnits)
}
if actual := contextEvidenceRefs(got.Contexts); !reflect.DeepEqual(actual, test.wantRefs) {
t.Fatalf("context evidence refs = %#v, want %#v", actual, test.wantRefs)
} }
}) })
} }
} }
func TestBuildIsStableAndOwnsSourceAndInputs(t *testing.T) { func TestBuildCopiesSelectedUnitsAndMetadata(t *testing.T) {
document := testDocument(t) document := testDocument(t)
refs := []source.SourceRef{ref(30, 30), ref(3, 3)} first, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
request := BuildRequest{
Source: document,
WindowUnits: 1,
SelectedLanes: []string{"spells", "npcs"},
LaneEvidence: []LaneEvidence{{LaneID: "spells", SourceRefs: refs}, {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
}
first, err := Build(request)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
secondRequest := request second, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
secondRequest.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}, {LaneID: "spells", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}}
second, err := Build(secondRequest)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(first, second) { if !reflect.DeepEqual(first[0], document.Units[0]) {
t.Fatalf("Build() order differs:\nfirst: %#v\nsecond: %#v", first, second) t.Fatalf("first unit = %#v, want unchanged source unit %#v", first[0], document.Units[0])
} }
first.SelectedLanes[0] = "changed" first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Build() returned metadata aliases to source document") t.Fatal("Build() returned metadata aliases to the source document")
} }
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later" document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Build() retained metadata aliases to source document") t.Fatal("Build() retained metadata aliases to the source document")
}
refs[0].StartUnitID = 999
if !containsEvidenceRef(second.Contexts[0].EvidenceRefs, ref(30, 30)) {
t.Fatal("Build() retained source-reference input aliases")
} }
} }
@@ -142,18 +72,11 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
want string want string
}{ }{
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"}, {name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
{name: "blank selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{" "} }, want: "selected_lanes"},
{name: "duplicate selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{"npcs", " npcs "} }, want: "duplicated"},
{name: "unselected contribution", mutate: func(request *BuildRequest) {
request.LaneEvidence = []LaneEvidence{{LaneID: "other", SourceRefs: []source.SourceRef{ref(3, 3)}}}
}, want: "not selected"},
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"}, {name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
{name: "invalid reference", mutate: func(request *BuildRequest) { {name: "invalid reference", mutate: func(request *BuildRequest) { request.SourceRefs = []source.SourceRef{ref(99, 99)} }, want: "source reference[0]"},
request.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(99, 99)}}}
}, want: "source reference[0]"},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
request := BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}} request := BuildRequest{Source: testDocument(t), SourceRefs: []source.SourceRef{ref(3, 3)}}
test.mutate(&request) test.mutate(&request)
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) { if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Build() error = %v, want %q", err, test.want) t.Fatalf("Build() error = %v, want %q", err, test.want)
@@ -162,7 +85,7 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
} }
} }
func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) { func TestCodecRoundTripsFixtureAndOwnsValues(t *testing.T) {
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json") fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -179,15 +102,16 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) { if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded) t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
} }
value.Contexts[0].Units[0].Text = "changed" value[0].Text = "changed"
decoded, err := codec.Decode(encoded) decoded, err := codec.Decode(encoded)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if decoded.Contexts[0].Units[0].Text != "The party meets Rowan." { if decoded[0].Text != "The party meets Rowan." {
t.Fatal("Encode() retained mutable document storage") t.Fatal("Encode() retained mutable document storage")
} }
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -203,82 +127,52 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed" first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Decode() returned metadata aliases") t.Fatal("Decode() returned metadata aliases")
} }
} }
func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) { func TestCodecRejectsInvalidDurablePayloads(t *testing.T) {
value, err := Build(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
if err != nil {
t.Fatal(err)
}
for _, test := range []struct { for _, test := range []struct {
name string name string
mutate func(*Document) content string
}{ }{
{name: "unsorted lanes", mutate: func(value *Document) { value.SelectedLanes = []string{"z", "a"} }}, {name: "null", content: "null"},
{name: "context range mismatch", mutate: func(value *Document) { value.Contexts[0].ContextRef.EndUnitID = 999 }}, {name: "wrapper object", content: `{"units":[]}`},
{name: "mismatched evidence source", mutate: func(value *Document) { value.Contexts[0].EvidenceRefs[0].SourceRef.SourceID = "other" }}, {name: "missing required unit field", content: `[{"id":1,"kind":"segment","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
{name: "invalid evidence range", mutate: func(value *Document) { {name: "unknown unit field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1},"unknown":true}]`},
value.Contexts[0].EvidenceRefs[0].SourceRef.StartUnitID = 10 {name: "unknown reference field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1,"unknown":true}}]`},
}}, {name: "invalid self reference", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":2}}]`},
{name: "duplicate context unit", mutate: func(value *Document) { value.Contexts = append(value.Contexts, value.Contexts[0]) }}, {name: "mixed source documents", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session-one","start_unit_id":1,"end_unit_id":1}},{"id":2,"kind":"segment","text":"two","ref":{"source_id":"session-two","start_unit_id":2,"end_unit_id":2}}]`},
{name: "duplicate units", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}},{"id":1,"kind":"segment","text":"two","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
{name: "multiple JSON values", content: `[] []`},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
candidate, err := clone(value) if _, err := New().Decode([]byte(test.content)); err == nil {
if err != nil {
t.Fatal(err)
}
test.mutate(&candidate)
if _, err := New().Encode(candidate); err == nil {
t.Fatal("Encode() error = nil, want durable model rejection")
}
})
}
content, err := New().Encode(value)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
name string
mutate func(map[string]any)
}{
{name: "missing contexts", mutate: func(value map[string]any) { delete(value, "contexts") }},
{name: "null contexts", mutate: func(value map[string]any) { value["contexts"] = nil }},
{name: "unknown fixed field", mutate: func(value map[string]any) { value["unknown"] = true }},
{name: "missing units", mutate: func(value map[string]any) { delete(contextObject(value, 0), "units") }},
{name: "null evidence refs", mutate: func(value map[string]any) { contextObject(value, 0)["evidence_refs"] = nil }},
} {
t.Run(test.name, func(t *testing.T) {
raw := decodeJSON(t, content)
test.mutate(raw)
mutated, err := json.Marshal(raw)
if err != nil {
t.Fatal(err)
}
if _, err := New().Decode(mutated); err == nil {
t.Fatal("Decode() error = nil, want strict payload rejection") t.Fatal("Decode() error = nil, want strict payload rejection")
} }
}) })
} }
if _, err := New().Decode(append(content, []byte(" {}")...)); err == nil { if _, err := New().Encode(nil); err == nil {
t.Fatal("Decode() error = nil, want trailing JSON rejection") t.Fatal("Encode(nil) error = nil, want array rejection")
} }
} }
func TestSerializeUsesFixedArtifactIdentity(t *testing.T) { func TestSerializeUsesFixedArtifactIdentityAndEmptyArray(t *testing.T) {
artifact, err := Serialize(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}}) artifact, err := Serialize(BuildRequest{Source: testDocument(t)})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion { if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact) t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
} }
if string(artifact.Content) != "[]" {
t.Fatalf("Serialize() content = %s, want []", artifact.Content)
}
decoded, err := New().Decode(artifact.Content) decoded, err := New().Decode(artifact.Content)
if err != nil || len(decoded.Contexts) != 0 || decoded.Contexts == nil { if err != nil || decoded == nil || len(decoded) != 0 {
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty contexts", decoded, err) t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty array", decoded, err)
} }
} }
@@ -306,45 +200,10 @@ func ref(start, end int) source.SourceRef {
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end} return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
} }
func contextUnitIDs(contexts []Context) [][]int { func unitIDs(units Document) []int {
values := make([][]int, len(contexts)) values := make([]int, len(units))
for index, context := range contexts { for index, unit := range units {
values[index] = make([]int, len(context.Units)) values[index] = unit.ID
for unitIndex, unit := range context.Units {
values[index][unitIndex] = unit.ID
}
} }
return values return values
} }
func contextEvidenceRefs(contexts []Context) [][]EvidenceRef {
values := make([][]EvidenceRef, len(contexts))
for index, context := range contexts {
values[index] = append([]EvidenceRef(nil), context.EvidenceRefs...)
}
return values
}
func containsEvidenceRef(values []EvidenceRef, want source.SourceRef) bool {
for _, value := range values {
if value.SourceRef == want {
return true
}
}
return false
}
func decodeJSON(t *testing.T, content []byte) map[string]any {
t.Helper()
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil {
t.Fatal(err)
}
return value
}
func contextObject(value map[string]any, index int) map[string]any {
return value["contexts"].([]any)[index].(map[string]any)
}

View File

@@ -14,37 +14,12 @@ const (
MediaType = "application/json" MediaType = "application/json"
) )
// Document is the durable union of direct evidence and surrounding source // Document is the durable selected source-unit excerpt.
// context selected for one accepted source document. type Document []source.SourceUnit
type Document struct {
SourceID string `json:"source_id"`
SourceDigest string `json:"source_digest"`
WindowUnits int `json:"window_units"`
SelectedLanes []string `json:"selected_lanes"`
Contexts []Context `json:"contexts"`
}
type Context struct { // BuildRequest supplies accepted source material and projected source references.
ContextRef source.SourceRef `json:"context_ref"`
EvidenceRefs []EvidenceRef `json:"evidence_refs"`
Units []source.SourceUnit `json:"units"`
}
type EvidenceRef struct {
LaneID string `json:"lane_id"`
SourceRef source.SourceRef `json:"source_ref"`
}
// LaneEvidence attributes direct source references to one selected lane.
type LaneEvidence struct {
LaneID string `json:"lane_id"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
// BuildRequest supplies accepted source material and direct lane evidence.
type BuildRequest struct { type BuildRequest struct {
Source *source.SourceDocument Source *source.SourceDocument
WindowUnits int WindowUnits int
SelectedLanes []string SourceRefs []source.SourceRef
LaneEvidence []LaneEvidence
} }

View File

@@ -1 +1 @@
{"source_id":"session-alpha","source_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","window_units":0,"selected_lanes":["npcs"],"contexts":[{"context_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13},"evidence_refs":[{"lane_id":"npcs","source_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}],"units":[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]}]} [{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]

View File

@@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/http" "net/http"
"os"
"regexp" "regexp"
"sort" "sort"
"strings" "strings"
@@ -115,6 +116,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
if err := validateOutputTarget(out); err != nil { if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err return contracts.StructuredCompletionResponse{}, err
} }
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
}
appendedMessages, err := promptKitCorrectionMessages(req.Correction)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion correction: %w", err)
}
promptID := strings.TrimSpace(req.PromptID) promptID := strings.TrimSpace(req.PromptID)
if promptID == "" { if promptID == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty") return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
@@ -136,6 +144,19 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
Inputs: promptKitInputs(req.Inputs), Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID), Vars: promptKitVars(req, sessionID),
Execution: execution, Execution: execution,
AppendedMessages: appendedMessages,
}
if req.StructuredOutputRepairAttempts != nil {
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("inspect PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
contract := inspection.OutputContract
contract.RepairAttempts = *req.StructuredOutputRepairAttempts
runReq.Validation = &contract
} }
prepared, err := c.engine.PrepareExecution(ctx, runReq) prepared, err := c.engine.PrepareExecution(ctx, runReq)
if err != nil { if err != nil {
@@ -169,6 +190,16 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
redactPromptKitError(err), redactPromptKitError(err),
) )
} }
if errors.Is(err, promptkit.ErrLLMGenerate) {
var generationErr *promptkit.GenerationError
status := 0
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
if errors.As(err, &generationErr) {
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)) return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
} }
if result == nil { if result == nil {
@@ -187,6 +218,41 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil return response, nil
} }
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
owned, err := contracts.CloneSemanticCorrection(correction)
if err != nil {
return nil, err
}
if owned == nil {
return nil, nil
}
return []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: string(owned.AssistantResponse)},
{Role: promptkit.RoleUser, Content: owned.UserGuidance},
}, nil
}
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
if generationErr == nil {
return nil
}
var secrets []string
if prepared != nil && strings.TrimSpace(prepared.EffectiveModelParams.APIKeyEnv) != "" {
if value, ok := os.LookupEnv(prepared.EffectiveModelParams.APIKeyEnv); ok {
secrets = append(secrets, value)
}
}
redact := func(value string) string {
return bearerTokenPattern.ReplaceAllString(RedactSecrets(value, secrets), "Bearer "+secretReplacement)
}
return &contracts.LLMDebugResponse{ProviderError: &contracts.LLMDebugProviderError{
StatusCode: generationErr.StatusCode(),
Code: redact(generationErr.ProviderCode()),
Type: redact(generationErr.ProviderType()),
Message: redact(generationErr.ProviderMessage()),
}}
}
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse { func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body content := result.Artifact.Body
if len(content) == 0 { if len(content) == 0 {
@@ -210,6 +276,7 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
PromptTokens: result.Usage.PromptTokens, PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens, CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens, TotalTokens: result.Usage.TotalTokens,
RepairAttempts: result.Validation.RepairAttempts,
Debug: promptKitDebugMaterial(prepared, result), Debug: promptKitDebugMaterial(prepared, result),
} }
} }

View File

@@ -1,9 +1,11 @@
package llm package llm
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"io" "io"
"io/fs" "io/fs"
"net/http" "net/http"
@@ -115,6 +117,74 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
} }
} }
func TestPromptKitClientAppendsSemanticCorrectionAfterRenderedPrompt(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
request := contracts.StructuredCompletionRequest{
PromptID: "adapter.direct-session",
ProfileID: "explicit-profile",
SessionID: "correction-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
Vars: map[string]any{"custom": "value"},
}
var ordinary map[string]any
if _, err := client.CompleteStructured(context.Background(), request, &ordinary); err != nil {
t.Fatalf("ordinary CompleteStructured() error = %v", err)
}
ordinaryMessages := append([]promptkit.RenderedMessage(nil), fake.lastRequest().Prompt.Messages...)
if len(ordinaryMessages) != 1 {
t.Fatalf("ordinary rendered messages = %#v, want only the declared prompt message", ordinaryMessages)
}
correction, err := contracts.NewSemanticCorrection([]byte(`{"previous":"response"}`), "Return the corrected JSON object.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
request.Correction = correction
var corrected map[string]any
if _, err := client.CompleteStructured(context.Background(), request, &corrected); err != nil {
t.Fatalf("corrected CompleteStructured() error = %v", err)
}
correctedMessages := fake.lastRequest().Prompt.Messages
if len(correctedMessages) != len(ordinaryMessages)+2 {
t.Fatalf("corrected message count = %d, want %d", len(correctedMessages), len(ordinaryMessages)+2)
}
if !reflect.DeepEqual(correctedMessages[:len(ordinaryMessages)], ordinaryMessages) {
t.Fatalf("ordinary rendered prefix changed: got %#v, want %#v", correctedMessages[:len(ordinaryMessages)], ordinaryMessages)
}
if got, want := correctedMessages[len(ordinaryMessages)], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"previous":"response"}`}); got != want {
t.Fatalf("assistant correction message = %#v, want %#v", got, want)
}
if got, want := correctedMessages[len(ordinaryMessages)+1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the corrected JSON object."}); got != want {
t.Fatalf("user correction message = %#v, want %#v", got, want)
}
}
func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *testing.T) {
const sensitiveResponse = "assistant-response-must-not-appear-in-errors"
for _, correction := range []*contracts.SemanticCorrection{
{AssistantResponse: []byte(sensitiveResponse), UserGuidance: " \t"},
{AssistantResponse: bytes.Repeat([]byte(sensitiveResponse), contracts.MaxAssistantResponseBytes/len(sensitiveResponse)+1), UserGuidance: "Use a smaller response."},
} {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{Correction: correction}, &out)
if err == nil || !strings.Contains(err.Error(), "correction") {
t.Fatalf("CompleteStructured() error = %v, want correction validation failure", err)
}
if strings.Contains(err.Error(), sensitiveResponse) {
t.Fatalf("correction validation error leaked response content: %v", err)
}
if got := atomic.LoadInt32(&fake.calls); got != 0 {
t.Fatalf("provider calls = %d, want no provider call after invalid correction", got)
}
}
}
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) { func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
const initialPrompt = `id: snapshot.test const initialPrompt = `id: snapshot.test
version: "v1" version: "v1"
@@ -456,6 +526,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second) t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
} }
}) })
t.Run("inherited parent", func(t *testing.T) {
profileDir := t.TempDir()
parentPath := filepath.Join(profileDir, "parent.yaml")
leafPath := filepath.Join(profileDir, "leaf.yaml")
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
}
})
} }
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) { func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
@@ -531,6 +627,130 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
} }
} }
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
if err != nil {
t.Fatal(err)
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
if err != nil {
t.Fatal(err)
}
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
if err != nil {
t.Fatal(err)
}
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
if err != nil {
t.Fatal(err)
}
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
}
}
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
if err != nil {
t.Fatal(err)
}
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
if err != nil {
t.Fatal(err)
}
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "" {
t.Fatalf("Authorization = %q, want omitted", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
if err != nil {
t.Fatal(err)
}
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
t.Fatal(err)
}
}
func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.T) {
const credential = "selected-test-credential"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":{"code":"temporary","type":"provider_error","message":"marker ` + credential + ` Bearer bearer-secret"}}`))
}))
defer server.Close()
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "generation-profile", SessionID: "generation-error-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
if !errors.Is(err, contracts.ErrLLMGeneration) {
t.Fatalf("error = %v", err)
}
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.StatusCode() != http.StatusServiceUnavailable {
t.Fatalf("generation error = %#v", generation)
}
if strings.Contains(err.Error(), credential) || strings.Contains(err.Error(), "marker") {
t.Fatalf("ordinary error leaked provider detail: %v", err)
}
if response.Debug == nil || response.Debug.Response == nil || response.Debug.Response.ProviderError == nil {
t.Fatalf("debug = %#v", response.Debug)
}
debug := response.Debug.Response.ProviderError
if debug.StatusCode != http.StatusServiceUnavailable || debug.Code != "temporary" || strings.Contains(debug.Message, credential) || strings.Contains(debug.Message, "bearer-secret") || !strings.Contains(debug.Message, "marker") {
t.Fatalf("debug provider error = %#v", debug)
}
}
func TestPromptKitClientRejectsInvalidRepairAttemptOverride(t *testing.T) {
for _, attempts := range []int{-1, 4} {
t.Run("invalid", func(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", StructuredOutputRepairAttempts: &attempts}, &out)
if err == nil || fake.calls != 0 {
t.Fatalf("error=%v calls=%d", err, fake.calls)
}
})
}
}
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) { func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
fingerprintFor := func(content string) CheckpointFingerprint { fingerprintFor := func(content string) CheckpointFingerprint {
t.Helper() t.Helper()
@@ -772,12 +992,15 @@ func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
} }
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`}) attempts := 1
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{{Content: `{"bad":true}`}, {Content: `{"bad":true}`}}}
client := newTestPromptKitClient(t, fake)
var out map[string]any var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test", PromptID: "adapter.test",
SessionID: "session-123", SessionID: "session-123",
StructuredOutputRepairAttempts: &attempts,
Inputs: contracts.LLMInputSet{ Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
}, },
@@ -785,6 +1008,9 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") { if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
t.Fatalf("CompleteStructured() error = %v, want validation failure", err) t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
} }
if got := atomic.LoadInt32(&fake.calls); got != 2 {
t.Fatalf("provider calls = %d, want exhausted repair budget", got)
}
if got := string(resp.Content); got != `{"bad":true}` { if got := string(resp.Content); got != `{"bad":true}` {
t.Fatalf("response content = %q, want raw failed output", got) t.Fatalf("response content = %q, want raw failed output", got)
} }
@@ -796,6 +1022,61 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
} }
} }
func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *testing.T) {
attempts := 1
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{
{Content: `{"bad":true}`, Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8}},
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
}}
client := newTestPromptKitClient(t, fake)
correction, err := contracts.NewSemanticCorrection([]byte(`{"bad":true}`), "Return the required ok field.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
var out struct {
OK bool `json:"ok"`
}
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
StructuredOutputRepairAttempts: &attempts,
SessionID: "repair-test",
Correction: correction,
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil || !out.OK {
t.Fatalf("CompleteStructured() = (%#v, %v), want repaired success", response, err)
}
if got := atomic.LoadInt32(&fake.calls); got != 2 {
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
}
requests := fake.requestsSnapshot()
if len(requests) != 2 || len(requests[0].Prompt.Messages) < 3 {
t.Fatalf("repair requests = %#v, want correction messages on the initial prepared request", requests)
}
messages := requests[0].Prompt.Messages
if got, want := messages[len(messages)-2], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"bad":true}`}); got != want {
t.Fatalf("repair assistant correction = %#v, want %#v", got, want)
}
if got, want := messages[len(messages)-1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the required ok field."}); got != want {
t.Fatalf("repair user correction = %#v, want %#v", got, want)
}
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
}
if response.Debug == nil || response.Debug.Prompt == nil || response.Debug.Response == nil {
t.Fatalf("debug = %#v, want prompt and response details", response.Debug)
}
if response.Debug.Prompt.OutputContract["repair_attempts"] != float64(1) ||
response.Debug.Response.Validation["repair_attempts"] != float64(1) ||
response.Debug.Response.Content != `{"ok":true}` ||
response.Debug.Response.Usage.TotalTokens != 26 {
t.Fatalf("debug repair result = %#v, want configured contract and repaired response", response.Debug)
}
}
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) { func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
@@ -847,8 +1128,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
t.Fatalf("error chain exposes credential: %v", err) t.Fatalf("error chain exposes credential: %v", err)
} }
} }
if errors.Unwrap(err) != nil { if !errors.Is(err, contracts.ErrLLMGeneration) {
t.Fatalf("provider failure must not expose a wrapped diagnostic: %v", err) t.Fatalf("provider failure = %v, want application generation classification", err)
} }
var recoveredProviderErr *credentialBearingProviderError var recoveredProviderErr *credentialBearingProviderError
if errors.As(err, &recoveredProviderErr) { if errors.As(err, &recoveredProviderErr) {
@@ -857,8 +1138,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
if errors.Is(err, promptkit.ErrLLMGenerate) { if errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err) t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
} }
if resp.Debug != nil { if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Response != nil {
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug) t.Fatalf("debug material = %#v, want prepared prompt without provider details", resp.Debug)
} }
} }
@@ -951,6 +1232,15 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
} }
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) { func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
attempts := 1
for _, test := range []struct {
name string
attempts *int
}{
{name: "during preparation"},
{name: "during override inspection", attempts: &attempts},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
@@ -958,6 +1248,7 @@ func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
var out map[string]any var out map[string]any
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
PromptID: "adapter.test", PromptID: "adapter.test",
StructuredOutputRepairAttempts: test.attempts,
Inputs: contracts.LLMInputSet{ Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
}, },
@@ -965,6 +1256,8 @@ func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("CompleteStructured() error = %v, want context canceled", err) t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
} }
})
}
} }
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) { func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
@@ -1071,6 +1364,66 @@ func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
wg.Wait() wg.Wait()
} }
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
attempts := 1
fake := &fakePromptKitLLM{
responses: []promptkit.GenerateResponse{
{Content: `{"bad":true}`},
{Content: `{"ok":true}`},
{Content: `{"ok":true}`},
},
block: make(chan struct{}),
}
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v", err)
}
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
firstDone := make(chan error, 1)
go func() {
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "repair-session",
StructuredOutputRepairAttempts: &attempts,
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
firstDone <- callErr
}()
waitForAtomicAtLeast(t, &fake.calls, 1)
secondDone := make(chan error, 1)
go func() {
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "queued-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
secondDone <- callErr
}()
close(fake.block)
if err := <-firstDone; err != nil {
t.Fatalf("repaired completion error = %v", err)
}
if err := <-secondDone; err != nil {
t.Fatalf("queued completion error = %v", err)
}
requests := fake.requestsSnapshot()
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
}
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
}
}
func TestPromptKitClientValidatesRequest(t *testing.T) { func TestPromptKitClientValidatesRequest(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any var out map[string]any
@@ -1225,11 +1578,13 @@ func (f *switchingPromptFS) resumePromptRead() {
type fakePromptKitLLM struct { type fakePromptKitLLM struct {
content string content string
responses []promptkit.GenerateResponse
allowEmpty bool allowEmpty bool
err error err error
block chan struct{} block chan struct{}
mu sync.Mutex mu sync.Mutex
last promptkit.GenerateRequest last promptkit.GenerateRequest
requests []promptkit.GenerateRequest
calls int32 calls int32
inFlight int32 inFlight int32
maxInFlight int32 maxInFlight int32
@@ -1244,8 +1599,9 @@ func (*credentialBearingProviderError) Error() string {
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.mu.Lock() f.mu.Lock()
f.last = req f.last = req
f.requests = append(f.requests, req)
f.mu.Unlock() f.mu.Unlock()
atomic.AddInt32(&f.calls, 1) call := atomic.AddInt32(&f.calls, 1)
current := atomic.AddInt32(&f.inFlight, 1) current := atomic.AddInt32(&f.inFlight, 1)
for { for {
seen := atomic.LoadInt32(&f.maxInFlight) seen := atomic.LoadInt32(&f.maxInFlight)
@@ -1264,6 +1620,14 @@ func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateR
if f.err != nil { if f.err != nil {
return nil, f.err return nil, f.err
} }
if len(f.responses) > 0 {
index := int(call - 1)
if index >= len(f.responses) {
return nil, fmt.Errorf("unexpected provider call %d", call)
}
response := f.responses[index]
return &response, nil
}
content := f.content content := f.content
if content == "" && !f.allowEmpty { if content == "" && !f.allowEmpty {
content = `{"ok":true}` content = `{"ok":true}`
@@ -1294,3 +1658,9 @@ func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
defer f.mu.Unlock() defer f.mu.Unlock()
return f.last return f.last
} }
func (f *fakePromptKitLLM) requestsSnapshot() []promptkit.GenerateRequest {
f.mu.Lock()
defer f.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), f.requests...)
}

View File

@@ -17,7 +17,7 @@ const (
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1" promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
// The built-in profile catalog is compiled into this pinned PromptKit // The built-in profile catalog is compiled into this pinned PromptKit
// release. Update this identity when the dependency is upgraded. // release. Update this identity when the dependency is upgraded.
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles" promptKitBuiltinProfileCatalogID = "promptkit:v0.9.0:builtin-profiles"
) )
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) { func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {

View File

@@ -89,6 +89,7 @@ const (
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused" CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused" CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step" CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
) )
type CheckpointDecision struct { type CheckpointDecision struct {
@@ -160,6 +161,8 @@ func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
return "accepted normalized artifact is reusable" return "accepted normalized artifact is reusable"
case CheckpointReasonRecomputeStep: case CheckpointReasonRecomputeStep:
return "selected step requires execution" return "selected step requires execution"
case CheckpointReasonValidationIncompleteLineage:
return "checkpoint reuse is disabled by validation-incomplete input lineage"
default: default:
return "checkpoint decision" return "checkpoint decision"
} }

View File

@@ -42,6 +42,7 @@ type debugTimedEnvelope struct {
LaneID string `json:"lane_id,omitempty"` LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"` ModuleKey string `json:"module_key,omitempty"`
Attempt int `json:"attempt,omitempty"` Attempt int `json:"attempt,omitempty"`
AttemptKind string `json:"attempt_kind,omitempty"`
StartedAt time.Time `json:"started_at"` StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"` CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"` DurationMS int64 `json:"duration_ms"`
@@ -110,24 +111,7 @@ type debugSerializedOutput struct {
Content debugBinaryEnvelope `json:"content"` Content debugBinaryEnvelope `json:"content"`
} }
type debugLLMInputMaterial struct { type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
Name string `json:"name"`
MediaType string `json:"media_type,omitempty"`
Content string `json:"content_base64,omitempty"`
Digest string `json:"digest,omitempty"`
OriginURI string `json:"origin_uri,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
}
type debugStructuredCompletionRequest struct {
StageName string `json:"stage_name"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
}
type debugStructuredCompletionResponse struct { type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"` Content string `json:"content,omitempty"`
@@ -160,9 +144,37 @@ type debugLLMCallReference struct {
PromptID string `json:"prompt_id,omitempty"` PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"` ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"` Error bool `json:"error,omitempty"`
} }
type debugValidationOutcome struct {
ValidatorName string `json:"validator_name"`
Outcome string `json:"outcome"`
AttemptCount int `json:"attempt_count"`
ReasonCode string `json:"reason_code,omitempty"`
}
type debugProducerAttempt struct {
Number int `json:"number"`
Kind string `json:"kind"`
Outcome string `json:"outcome"`
Validation []debugValidationOutcome `json:"validation,omitempty"`
}
type debugProducerTerminal struct {
ProducerAttemptCount int `json:"producer_attempt_count"`
Attempts []debugProducerAttempt `json:"attempts,omitempty"`
AggregateReasonCodes []string `json:"aggregate_reason_codes,omitempty"`
ValidationComplete bool `json:"validation_complete"`
EffectivePolicy ValidationPolicy `json:"effective_policy"`
TerminalAction string `json:"terminal_action"`
Summary artifacts.ValidationSummary `json:"validation_summary"`
}
type debugValidationCall struct { type debugValidationCall struct {
ValidatorName string `json:"validator_name"` ValidatorName string `json:"validator_name"`
Request any `json:"request"` Request any `json:"request"`
@@ -267,6 +279,10 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
PromptID: req.PromptID, PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID), ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)), Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil, Error: err != nil,
} }
if scope := debugLLMScopeFromContext(ctx); scope != nil { if scope := debugLLMScopeFromContext(ctx); scope != nil {
@@ -423,6 +439,50 @@ func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
return terminalErr return terminalErr
} }
func debugValidationOutcomes(report validationReport) []debugValidationOutcome {
if len(report.records) == 0 {
return nil
}
outcomes := make([]debugValidationOutcome, 0, len(report.records))
for _, record := range report.records {
outcomes = append(outcomes, debugValidationOutcome{
ValidatorName: record.validatorName,
Outcome: string(record.outcome),
AttemptCount: record.attemptCount,
ReasonCode: record.reasonCode,
})
}
return outcomes
}
func writeProducerTerminalDebug(recorder DebugRecorder, name string, terminal producerAttemptTerminal, policy ValidationPolicy, summary artifacts.ValidationSummary) error {
attempts := make([]debugProducerAttempt, 0, len(terminal.Provenance))
for _, item := range terminal.Provenance {
attempts = append(attempts, debugProducerAttempt{
Number: item.Number,
Kind: string(item.Kind),
Outcome: string(item.Outcome),
Validation: debugValidationOutcomes(item.Validation),
})
}
return writeDebugTimed(recorder, name, debugTimedEnvelope{
Stage: summary.Stage,
StepID: summary.StepID,
LaneID: summary.LaneID,
ModuleKey: summary.ModuleKey,
StartedAt: time.Now().UTC(),
Payload: debugProducerTerminal{
ProducerAttemptCount: summary.ProducerAttemptCount,
Attempts: attempts,
AggregateReasonCodes: append([]string(nil), summary.ReasonCodes...),
ValidationComplete: summary.Status == "complete",
EffectivePolicy: policy,
TerminalAction: string(terminal.Action),
Summary: artifacts.CloneValidationSummary(summary),
},
})
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope { func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
content = redactSecretBytes(content) content = redactSecretBytes(content)
return debugBinaryEnvelope{ return debugBinaryEnvelope{
@@ -567,29 +627,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
} }
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest { func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs)) return req.DebugSummary()
for key, material := range req.Inputs {
inputs[key] = debugLLMInputMaterial{
Name: material.Name,
MediaType: material.MediaType,
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
Digest: material.Digest,
OriginURI: material.OriginURI,
SizeBytes: material.SizeBytes,
}
}
if len(inputs) == 0 {
inputs = nil
}
return debugStructuredCompletionRequest{
StageName: req.StageName,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
Inputs: inputs,
Vars: redactSensitiveMap(req.Vars),
}
} }
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse { func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
@@ -691,6 +729,7 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult { func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message))) result.Message = string(redactSecretBytes([]byte(result.Message)))
result.CorrectionGuidance = ""
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath))) result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
for i := range result.Warnings { for i := range result.Warnings {
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message))) result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))

View File

@@ -47,6 +47,32 @@ func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) {
} }
} }
func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) {
const assistantResponse = `{"secret":"assistant response"}`
const userGuidance = "secret user guidance"
correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance)
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
summary := debugCompletionRequest(contracts.StructuredCompletionRequest{
Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
Vars: map[string]any{"custom": "value"},
Correction: correction,
})
encoded, err := json.Marshal(summary)
if err != nil {
t.Fatalf("marshal completion summary: %v", err)
}
for _, secret := range []string{assistantResponse, userGuidance} {
if strings.Contains(string(encoded), secret) {
t.Fatalf("debug completion summary leaked %q: %s", secret, encoded)
}
}
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary)
}
}
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) { func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
doc := validSourceDocument() doc := validSourceDocument()
envelope := debugSourceDocumentEnvelope(doc) envelope := debugSourceDocumentEnvelope(doc)

View File

@@ -20,7 +20,6 @@ type debugEvidenceContextSummary struct {
SchemaVersion string `json:"schema_version"` SchemaVersion string `json:"schema_version"`
SelectedLanes []string `json:"selected_lanes"` SelectedLanes []string `json:"selected_lanes"`
WindowUnits int `json:"window_units"` WindowUnits int `json:"window_units"`
ContextCount int `json:"context_count"`
UnitCount int `json:"unit_count"` UnitCount int `json:"unit_count"`
SourceDigest string `json:"source_digest"` SourceDigest string `json:"source_digest"`
} }
@@ -51,8 +50,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
request := evidencecontext.BuildRequest{ request := evidencecontext.BuildRequest{
Source: doc, Source: doc,
WindowUnits: prepared.evidencePlan.policy.WindowUnits, WindowUnits: prepared.evidencePlan.policy.WindowUnits,
SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...), SourceRefs: make([]source.SourceRef, 0),
LaneEvidence: make([]evidencecontext.LaneEvidence, 0, len(prepared.evidencePlan.lanes)),
} }
for _, lane := range prepared.evidencePlan.lanes { for _, lane := range prepared.evidencePlan.lanes {
output, ok := byLane[lane.laneID] output, ok := byLane[lane.laneID]
@@ -73,10 +71,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID) return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID)
} }
request.LaneEvidence = append(request.LaneEvidence, evidencecontext.LaneEvidence{ request.SourceRefs = append(request.SourceRefs, references...)
LaneID: lane.laneID,
SourceRefs: append([]source.SourceRef(nil), references...),
})
} }
document, err := evidencecontext.Build(request) document, err := evidencecontext.Build(request)
@@ -99,13 +94,10 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
SchemaID: artifact.Schema.ID, SchemaID: artifact.Schema.ID,
SchemaName: artifact.Schema.Name, SchemaName: artifact.Schema.Name,
SchemaVersion: artifact.Schema.Version, SchemaVersion: artifact.Schema.Version,
SelectedLanes: append([]string(nil), document.SelectedLanes...), SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...),
WindowUnits: document.WindowUnits, WindowUnits: prepared.evidencePlan.policy.WindowUnits,
ContextCount: len(document.Contexts), SourceDigest: doc.Digest,
SourceDigest: document.SourceDigest, UnitCount: len(document),
}
for _, context := range document.Contexts {
summary.UnitCount += len(context.Units)
} }
return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil
} }

View File

@@ -97,14 +97,11 @@ func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs(t *testing.T)
} }
value := decodeCapturedEvidence(t, encoder) value := decodeCapturedEvidence(t, encoder)
if !reflect.DeepEqual(value.SelectedLanes, []string{"alpha", "beta", "inactive"}) || len(value.Contexts) != 1 || len(value.Contexts[0].Units) != 3 { if actual := []int{value[0].ID, value[1].ID, value[2].ID}; !reflect.DeepEqual(actual, []int{1, 2, 3}) {
t.Fatalf("evidence context = %#v, want selected union", value) t.Fatalf("evidence context = %#v, want selected union", value)
} }
if got := value.Contexts[0].EvidenceRefs; len(got) != 2 || got[0].LaneID != "alpha" || got[1].LaneID != "beta" {
t.Fatalf("evidence refs = %#v, want both selected lanes", got)
}
debugJSON := string(debug.json["output/evidence-context.json"]) debugJSON := string(debug.json["output/evidence-context.json"])
if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || !strings.Contains(debugJSON, `"context_count":1`) || !strings.Contains(debugJSON, `"unit_count":3`) { if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || strings.Contains(debugJSON, "context_count") || !strings.Contains(debugJSON, `"unit_count":3`) {
t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON) t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON)
} }
} }
@@ -117,9 +114,10 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{ prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
}, },
}} }}
prepared.Steps[1].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
installEvidencePlan(prepared, 0, []string{"absent", "present", "rejected"}, func(notes codecNotes) ([]source.SourceRef, error) { installEvidencePlan(prepared, 0, []string{"absent", "present", "rejected"}, func(notes codecNotes) ([]source.SourceRef, error) {
if len(notes.Items) > 0 && notes.Items[0] == "present" { if len(notes.Items) > 0 && notes.Items[0] == "present" {
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil
@@ -134,7 +132,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected) t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected)
} }
value := decodeCapturedEvidence(t, encoder) value := decodeCapturedEvidence(t, encoder)
if len(value.Contexts) != 1 || len(value.Contexts[0].EvidenceRefs) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "present" { if len(value) != 1 || value[0].ID != 1 {
t.Fatalf("evidence context = %#v, want present lane only", value) t.Fatalf("evidence context = %#v, want present lane only", value)
} }
} }
@@ -232,7 +230,7 @@ func TestRunnerEvidenceContextRebuildsFromAcceptedCheckpoint(t *testing.T) {
t.Fatalf("Run() error = %v", err) t.Fatalf("Run() error = %v", err)
} }
value := decodeCapturedEvidence(t, encoder) value := decodeCapturedEvidence(t, encoder)
if len(value.Contexts) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "notes" { if len(value) != 1 || value[0].ID != 1 {
t.Fatalf("evidence context = %#v, want checkpointed normalized output", value) t.Fatalf("evidence context = %#v, want checkpointed normalized output", value)
} }
} }

View File

@@ -58,11 +58,20 @@ func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpe
if !ok { if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation) return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
} }
correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone extraction correction: %w", err)
}
request.Correction = correction
result, err := extractor.Extract(ctx, request) result, err := extractor.Extract(ctx, request)
if err != nil { if err != nil {
return erasedTypedResult{}, err return erasedTypedResult{}, err
} }
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone extraction model candidate: %w", err)
}
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), ModelCandidate: candidate}, nil
}} }}
if registry.typedEntries == nil { if registry.typedEntries == nil {
registry.typedEntries = map[string]typedExtractorEntry{} registry.typedEntries = map[string]typedExtractorEntry{}

View File

@@ -41,6 +41,54 @@ func operationReferenceSet(input RunInput, target ResolvedReferenceTarget) contr
return CloneReferenceSet(target.ReferenceSet) return CloneReferenceSet(target.ReferenceSet)
} }
// referenceTargetReuseEligible reports whether every generated artifact in a
// stage's reference set descends exclusively from fully validated work. Static
// references and callers that do not supply lineage metadata are reusable.
func referenceTargetReuseEligible(input RunInput, target ResolvedReferenceTarget) bool {
if input.referenceReuseEligibility == nil {
return true
}
eligible, ok := input.referenceReuseEligibility[keyForReferenceTarget(target)]
return !ok || eligible
}
func laneReferencesReuseEligible(input RunInput, lane ResolvedArtifactLane) bool {
return referenceTargetReuseEligible(input, lane.ExtractReferences) &&
referenceTargetReuseEligible(input, lane.MergeReferences) &&
referenceTargetReuseEligible(input, lane.NormalizeReferences)
}
// buildStepReferenceReuseEligibility carries validation completeness alongside
// generated references without exposing the internal lineage flag in artifact
// payloads. A target becomes ineligible when any generated input is ineligible.
func buildStepReferenceReuseEligibility(step PreparedPipelineStep, outputs map[generatedOutputKey]bool) map[referenceTargetKey]bool {
eligibility := make(map[referenceTargetKey]bool)
for _, prepared := range step.lanes {
lane := prepared.resolved
for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
generated := false
eligible := true
for _, binding := range target.Bindings {
if binding.Artifact == nil {
continue
}
generated = true
producer := generatedOutputKeyFor(binding.Artifact.Step, binding.Artifact.Lane)
if reusable, ok := outputs[producer]; ok && !reusable {
eligible = false
}
}
if generated {
eligibility[keyForReferenceTarget(target)] = eligible
}
}
}
if len(eligibility) == 0 {
return nil
}
return eligibility
}
// buildStepReferenceSets resolves every generated binding for a step before // buildStepReferenceSets resolves every generated binding for a step before
// any lane in that step is allowed to start. Each returned set is a fresh // any lane in that step is allowed to start. Each returned set is a fresh
// operation-time view; prepared reference sets are never modified. // operation-time view; prepared reference sets are never modified.

View File

@@ -85,11 +85,19 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
} }
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value} outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
} }
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata}) correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone merge correction: %w", err)
}
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
if err != nil { if err != nil {
return erasedTypedResult{}, err return erasedTypedResult{}, err
} }
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone merge model candidate: %w", err)
}
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), ModelCandidate: candidate}, nil
}, },
} }
return nil return nil

View File

@@ -24,6 +24,7 @@ type ModuleSpec struct {
Key string Key string
Stage ModuleStage Stage ModuleStage
ExecutionClass contracts.ExecutionClass ExecutionClass contracts.ExecutionClass
CorrectionProtocol contracts.CorrectionProtocol
ArtifactKind contracts.ArtifactKind ArtifactKind contracts.ArtifactKind
Provides []string Provides []string
Requires []string Requires []string
@@ -32,10 +33,12 @@ type ModuleSpec struct {
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec { func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass))) executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
correctionProtocol := contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol)))
return ModuleSpec{ return ModuleSpec{
Key: strings.TrimSpace(spec.Key), Key: strings.TrimSpace(spec.Key),
Stage: spec.Stage, Stage: spec.Stage,
ExecutionClass: executionClass, ExecutionClass: executionClass,
CorrectionProtocol: correctionProtocol,
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind), ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
Provides: normalizeCapabilities(spec.Provides), Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires), Requires: normalizeCapabilities(spec.Requires),
@@ -73,6 +76,7 @@ func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
Key: spec.Key, Key: spec.Key,
Stage: spec.Stage, Stage: spec.Stage,
ExecutionClass: spec.ExecutionClass, ExecutionClass: spec.ExecutionClass,
CorrectionProtocol: spec.CorrectionProtocol,
ArtifactKind: spec.ArtifactKind, ArtifactKind: spec.ArtifactKind,
Provides: append([]string(nil), spec.Provides...), Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...), Requires: append([]string(nil), spec.Requires...),
@@ -93,6 +97,19 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked { if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass) return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
} }
if spec.CorrectionProtocol != "" {
if err := spec.CorrectionProtocol.Validate(); err != nil {
return fmt.Errorf("%s %q correction protocol: %w", kind, spec.Key, err)
}
if spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return fmt.Errorf("%s %q correction protocol requires an LLM-backed execution class", kind, spec.Key)
}
switch spec.Stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return fmt.Errorf("%s %q correction protocol is not supported for %q stage", kind, spec.Key, spec.Stage)
}
}
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize { if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key) return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
} }

View File

@@ -32,14 +32,70 @@ func TestValidateModuleSpecRequiresSupportedExecutionClass(t *testing.T) {
} }
} }
func TestCloneModuleSpecPreservesExecutionClass(t *testing.T) { func TestCloneModuleSpecPreservesCorrectionProtocol(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{Key: " module ", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}) spec := normalizeModuleSpec(ModuleSpec{
Key: " module ",
Stage: StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: " single_response_v1 ",
})
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
t.Fatalf("normalized CorrectionProtocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
}
cloned := cloneModuleSpec(spec) cloned := cloneModuleSpec(spec)
if !reflect.DeepEqual(cloned, spec) { if !reflect.DeepEqual(cloned, spec) {
t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec) t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec)
} }
} }
func TestValidateModuleSpecCorrectionProtocolEligibility(t *testing.T) {
tests := []struct {
name string
stage ModuleStage
class contracts.ExecutionClass
protocol contracts.CorrectionProtocol
want string
}{
{name: "LLM chunk", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
{name: "LLM extract", stage: StageExtract, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
{name: "LLM merge", stage: StageMerge, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
{name: "LLM normalize", stage: StageNormalize, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
{name: "deterministic chunk", stage: StageChunk, class: contracts.ExecutionClassDeterministic, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "LLM-backed"},
{name: "input", stage: StageInput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
{name: "output", stage: StageOutput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
{name: "unknown protocol", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: "unsupported", want: "unsupported correction protocol"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ExecutionClass: test.class,
CorrectionProtocol: test.protocol,
})
err := validateModuleSpec("module", test.stage, spec)
if test.want == "" && err != nil {
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
}
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
t.Fatalf("validateModuleSpec() error = %v, want %q", err, test.want)
}
})
}
}
func TestNormalizeValidatorSpecRejectsCorrectionProtocol(t *testing.T) {
_, err := normalizeValidatorSpec(ValidatorSpec{
Key: "validator",
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
})
if err == nil || !strings.Contains(err.Error(), "correction protocol") {
t.Fatalf("normalizeValidatorSpec() error = %v, want correction protocol error", err)
}
}
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) { func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -76,11 +76,19 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
if err != nil { if err != nil {
return erasedTypedResult{}, err return erasedTypedResult{}, err
} }
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata}) correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone normalize correction: %w", err)
}
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
if err != nil { if err != nil {
return erasedTypedResult{}, err return erasedTypedResult{}, err
} }
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), Retry: cloneNormalizeRetry(result.Retry)}, nil candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
if err != nil {
return erasedTypedResult{}, fmt.Errorf("clone normalize model candidate: %w", err)
}
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), Retry: cloneNormalizeRetry(result.Retry), ModelCandidate: candidate}, nil
}, },
} }
return nil return nil

View File

@@ -15,6 +15,7 @@ import (
type PreparedPipeline struct { type PreparedPipeline struct {
Input ModuleBinding Input ModuleBinding
Chunk ModuleBinding Chunk ModuleBinding
ChunkCorrectionProtocol contracts.CorrectionProtocol
Steps []PreparedPipelineStep Steps []PreparedPipelineStep
Output ModuleBinding Output ModuleBinding
@@ -37,6 +38,9 @@ type PreparedPipelineStep struct {
type PreparedArtifactLane struct { type PreparedArtifactLane struct {
Resolved ResolvedArtifactLane Resolved ResolvedArtifactLane
ExtractCorrectionProtocol contracts.CorrectionProtocol
MergeCorrectionProtocol contracts.CorrectionProtocol
NormalizeCorrectionProtocol contracts.CorrectionProtocol
} }
type preparedLaneExecutor struct { type preparedLaneExecutor struct {
@@ -79,6 +83,7 @@ type preparedValidator struct {
typedValidate typedValidateOperation typedValidate typedValidateOperation
chunk contracts.ChunkValidator chunk contracts.ChunkValidator
serialized contracts.SerializedValidator serialized contracts.SerializedValidator
position int
} }
// Prepare validates all configured options and constructs every selected // Prepare validates all configured options and constructs every selected
@@ -87,6 +92,9 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
if err := validateResolvedPipeline(resolved); err != nil { if err := validateResolvedPipeline(resolved); err != nil {
return nil, err return nil, err
} }
if err := validateCorrectionRetryCapabilities(resolved); err != nil {
return nil, err
}
if err := validateRegistrySet(resolved, registries); err != nil { if err := validateRegistrySet(resolved, registries); err != nil {
return nil, err return nil, err
} }
@@ -94,6 +102,7 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
prepared := &PreparedPipeline{ prepared := &PreparedPipeline{
Input: cloneModuleBinding(stable.Input), Input: cloneModuleBinding(stable.Input),
Chunk: cloneModuleBinding(stable.Chunk), Chunk: cloneModuleBinding(stable.Chunk),
ChunkCorrectionProtocol: stable.ChunkCorrectionProtocol,
Output: cloneModuleBinding(stable.Output), Output: cloneModuleBinding(stable.Output),
resolved: stable, resolved: stable,
dependencies: deps, dependencies: deps,
@@ -131,7 +140,12 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
if err != nil { if err != nil {
return nil, err return nil, err
} }
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{Resolved: cloneResolvedArtifactLane(lane)}) preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{
Resolved: cloneResolvedArtifactLane(lane),
ExtractCorrectionProtocol: lane.ExtractCorrectionProtocol,
MergeCorrectionProtocol: lane.MergeCorrectionProtocol,
NormalizeCorrectionProtocol: lane.NormalizeCorrectionProtocol,
})
preparedStep.lanes = append(preparedStep.lanes, executor) preparedStep.lanes = append(preparedStep.lanes, executor)
} }
prepared.Steps[stepIndex] = preparedStep prepared.Steps[stepIndex] = preparedStep
@@ -199,6 +213,40 @@ func prepareEvidencePlan(resolved ResolvedPipeline, registries Registries, outpu
return plan, nil return plan, nil
} }
func validateCorrectionRetryCapabilities(pipeline ResolvedPipeline) error {
validate := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass, protocol contracts.CorrectionProtocol) error {
if executionClass != contracts.ExecutionClassLLMBacked || binding.Retries == 0 {
return nil
}
chain := resolvedValidatorChain(stage, laneID, binding.Module, pipeline.ValidatorChains)
if len(chain.Validators) == 0 || protocol == contracts.CorrectionProtocolSingleResponseV1 {
return nil
}
if laneID == "" {
return fmt.Errorf("pipeline %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
}
return fmt.Errorf("pipeline %q lane %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, laneID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
}
if err := validate(StageChunk, "", pipeline.Chunk, pipeline.ChunkExecutionClass, pipeline.ChunkCorrectionProtocol); err != nil {
return err
}
for _, step := range pipeline.Steps {
for _, lane := range step.ArtifactLanes {
if err := validate(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass, lane.ExtractCorrectionProtocol); err != nil {
return err
}
if err := validate(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass, lane.MergeCorrectionProtocol); err != nil {
return err
}
if err := validate(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizeCorrectionProtocol); err != nil {
return err
}
}
}
return nil
}
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) { func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)} executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest { request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
@@ -406,6 +454,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline { func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
out := in out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)

View File

@@ -0,0 +1,391 @@
package pipeline
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// producerAttemptKind records why a producer invocation followed the prior
// one. It is deliberately independent of any artifact family.
type producerAttemptKind string
const (
producerAttemptInitial producerAttemptKind = "initial"
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
)
type producerAttemptOutcome string
const (
producerAttemptAccepted producerAttemptOutcome = "accepted"
producerAttemptRejected producerAttemptOutcome = "rejected"
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
producerAttemptRetried producerAttemptOutcome = "retried"
producerAttemptFailed producerAttemptOutcome = "failed"
)
type producerTerminalAction string
const (
producerTerminalAccepted producerTerminalAction = "accepted"
producerTerminalRejected producerTerminalAction = "reject_output"
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
producerTerminalFailed producerTerminalAction = "fail_run"
)
// producerAttemptRequest contains only attempt-local control material. The
// producer reconstructs its ordinary request from its own durable inputs.
type producerAttemptRequest struct {
Number int
Kind producerAttemptKind
Correction *contracts.SemanticCorrection
}
// producerRetryDirective asks for another producer invocation while retaining
// the current value as a safe fallback if its shared budget is exhausted.
// Artifact-specific adapters are responsible for validating and populating it.
type producerRetryDirective struct {
FallbackWarnings []contracts.Warning
}
func (directive *producerRetryDirective) clone() *producerRetryDirective {
if directive == nil {
return nil
}
return &producerRetryDirective{FallbackWarnings: cloneWarnings(directive.FallbackWarnings)}
}
// producerAttemptOutput is intentionally artifact-neutral. Value remains
// opaque to the state machine; Candidate is the attempt-local response that
// may support semantic correction.
type producerAttemptOutput struct {
Value any
Candidate *contracts.ModelCandidate
Warnings []contracts.Warning
Retry *producerRetryDirective
}
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
candidate, err := contracts.CloneModelCandidate(output.Candidate)
if err != nil {
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
}
return producerAttemptOutput{
Value: output.Value,
Candidate: candidate,
Warnings: cloneWarnings(output.Warnings),
Retry: output.Retry.clone(),
}, nil
}
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
type producerAttemptConfig struct {
Retries int
Policy ValidationPolicy
AllowStructuralRetry bool
}
// producerAttemptProvenance is ordered by producer invocation. It retains the
// settled validation report for later debug and durable-provenance adapters.
type producerAttemptProvenance struct {
Number int
Kind producerAttemptKind
Outcome producerAttemptOutcome
Validation validationReport
}
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
provenance.Validation = cloneValidationReport(provenance.Validation)
return provenance
}
// producerAttemptTerminal describes the state-machine decision without
// materializing an artifact or persisting stage-specific diagnostics.
type producerAttemptTerminal struct {
Action producerTerminalAction
Value any
Warnings []contracts.Warning
Rejection *contracts.RejectedOutput
Validation validationReport
ValidationIncomplete bool
Provenance []producerAttemptProvenance
}
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
terminal.Warnings = cloneWarnings(terminal.Warnings)
if terminal.Rejection != nil {
rejection := *terminal.Rejection
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
terminal.Rejection = &rejection
}
terminal.Validation = cloneValidationReport(terminal.Validation)
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
return terminal
}
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
if len(provenance) == 0 {
return nil
}
cloned := make([]producerAttemptProvenance, len(provenance))
for index, item := range provenance {
cloned[index] = item.clone()
}
return cloned
}
func cloneValidationReport(report validationReport) validationReport {
return validationReport{records: report.Records()}
}
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
if ctx == nil {
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
}
if config.Retries < 0 {
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
}
if produce == nil {
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
}
if validate == nil {
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
}
attemptLimit := config.Retries + 1
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
kind := producerAttemptInitial
var correction *contracts.SemanticCorrection
for number := 1; number <= attemptLimit; number++ {
if err := ctx.Err(); err != nil {
return failedProducerAttempt(provenance), err
}
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
if err != nil {
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
}
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
if isImmediateProducerFailure(err) {
return failedProducerAttempt(provenance), err
}
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
if number < attemptLimit {
kind, correction = producerAttemptStructuralRetry, nil
continue
}
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
}
if number < attemptLimit {
kind, correction = producerAttemptOperationalRetry, nil
continue
}
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
}
output, err = output.clone()
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
if output.Retry != nil && number < attemptLimit {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
kind, correction = producerAttemptModuleRetry, nil
continue
}
if output.Retry != nil {
output.Warnings = append(output.Warnings, cloneWarnings(output.Retry.FallbackWarnings)...)
}
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
}
report, err := validate(ctx, output)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
report = cloneValidationReport(report)
if err := ctx.Err(); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), err
}
if rejection := report.FirstRejection(); rejection != nil {
if correctionCandidate != nil {
if err := correctionCandidate.Validate(); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
}
}
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
correctionRequest, guidanceErr := report.CorrectionRequest()
if guidanceErr != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
}
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
kind = producerAttemptSemanticRetry
continue
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
}
if incomplete := firstIncompleteValidation(report); incomplete != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
warnings := terminalWarnings(output, report)
warnings = append(warnings, incompleteValidationWarnings(report)...)
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Warnings: warnings, Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
}
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Warnings: terminalWarnings(output, report), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
}
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
}
func isImmediateProducerFailure(err error) bool {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
var debugErr *attemptDebugPersistenceError
return errors.As(err, &debugErr)
}
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
switch policy.ProducerStructuralFailure {
case ProducerStructuralFailureRejectOutput:
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
case ProducerStructuralFailureFailRun:
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
default:
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
}
}
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
switch policy.SemanticRejection {
case SemanticRejectionRejectOutput:
return producerAttemptTerminal{Action: producerTerminalRejected, Warnings: terminalWarnings(output, report), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
case SemanticRejectionFailRun:
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
default:
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
}
}
func firstIncompleteValidation(report validationReport) *validationRecord {
for _, record := range report.records {
if record.outcome == validationFailed || record.outcome == validationSkipped {
clone := record.clone()
return &clone
}
}
return nil
}
func terminalWarnings(output producerAttemptOutput, report validationReport) []contracts.Warning {
warnings := cloneWarnings(output.Warnings)
warnings = append(warnings, report.Warnings()...)
return warnings
}
// incompleteValidationWarnings reports only validators that exhausted their
// execution budget. It never reports rejected candidates, and it uses fixed
// text so provider errors and correction content cannot cross this boundary.
func incompleteValidationWarnings(report validationReport) []contracts.Warning {
warnings := make([]contracts.Warning, 0)
for _, record := range report.records {
if record.outcome != validationFailed {
continue
}
warnings = append(warnings, contracts.Warning{
Scope: record.validatorName,
ReasonCode: "validator_execution_incomplete",
Message: "Validator execution did not complete within its configured budget.",
})
}
return warnings
}
// validationSummary projects a terminal state-machine result into the durable
// bounded representation used by manifests, rejections, and receipts.
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
summary := artifacts.ValidationSummary{
Stage: string(stage),
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
ChunkID: chunkID,
ChunkIndex: chunkIndex,
ProducerAttemptCount: len(terminal.Provenance),
TerminalAction: string(terminal.Action),
}
switch {
case terminal.Action == producerTerminalRejected:
summary.Status = "rejected"
case terminal.Action == producerTerminalFailed:
summary.Status = "incomplete"
case terminal.ValidationIncomplete:
summary.Status = "incomplete"
default:
summary.Status = "complete"
}
seenValidators := make(map[string]struct{})
seenReasons := make(map[string]struct{})
seenIncomplete := make(map[string]struct{})
for _, record := range terminal.Validation.records {
if record.reasonCode != "" {
if _, exists := seenReasons[record.reasonCode]; !exists {
seenReasons[record.reasonCode] = struct{}{}
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
}
}
if record.outcome == validationRejected {
if _, exists := seenValidators[record.validatorName]; !exists {
seenValidators[record.validatorName] = struct{}{}
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
}
}
if record.outcome == validationFailed || record.outcome == validationSkipped {
if _, exists := seenIncomplete[record.validatorName]; !exists {
seenIncomplete[record.validatorName] = struct{}{}
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
}
}
}
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
}
}
return summary
}
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
}

View File

@@ -0,0 +1,414 @@
package pipeline
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) {
candidate := attemptCandidate(t, "first response")
producerCalls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
producerCalls++
switch request.Number {
case 1:
return producerAttemptOutput{}, errors.New("temporary producer failure")
case 2:
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
case 3:
return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil
default:
t.Fatalf("unexpected producer attempt %d", request.Number)
return producerAttemptOutput{}, nil
}
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
t.Fatalf("terminal = %#v, want accepted value", terminal)
}
if producerCalls != 3 {
t.Fatalf("producer calls = %d, want 3", producerCalls)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
}
func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) {
first := attemptCandidate(t, "first response")
second := attemptCandidate(t, "second response")
var corrections []*contracts.SemanticCorrection
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Correction != nil {
corrections = append(corrections, request.Correction)
}
switch request.Number {
case 1:
return producerAttemptOutput{Value: "one", Candidate: first}, nil
case 2:
return producerAttemptOutput{Value: "two", Candidate: second}, nil
case 3:
return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil
default:
t.Fatalf("unexpected producer attempt %d", request.Number)
return producerAttemptOutput{}, nil
}
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "three" {
return validationReport{}, nil
}
return rejectedAttemptReport("semantic_defect"), nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted {
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
}
if len(corrections) != 2 {
t.Fatalf("correction count = %d, want 2", len(corrections))
}
if corrections[0] == corrections[1] {
t.Fatal("semantic corrections reused the same pointer")
}
if got := string(corrections[0].AssistantResponse); got != "first response" {
t.Fatalf("first correction response = %q", got)
}
if got := string(corrections[1].AssistantResponse); got != "second response" {
t.Fatalf("second correction response = %q", got)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
}
func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) {
tests := []struct {
name string
config producerAttemptConfig
produce producerAttemptProducer
validate producerAttemptValidator
wantAction producerTerminalAction
wantError bool
wantValue any
}{
{
name: "structural reject output",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
},
validate: approveAttempt,
wantAction: producerTerminalRejected,
},
{
name: "structural fail run",
config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
},
validate: approveAttempt,
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "operational failure after retry exhaustion",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, errors.New("producer unavailable")
},
validate: approveAttempt,
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "semantic reject output",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("bad"), nil
},
wantAction: producerTerminalRejected,
},
{
name: "semantic fail run",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("bad"), nil
},
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "validator failure warns and continues",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "kept"}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return failedAttemptReport(), nil
},
wantAction: producerTerminalIncompleteAccepted,
wantValue: "kept",
},
{
name: "validator failure fails run",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return failedAttemptReport(), nil
},
wantAction: producerTerminalFailed,
wantError: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate)
if (err != nil) != test.wantError {
t.Fatalf("error = %v, want error %t", err, test.wantError)
}
if terminal.Action != test.wantAction {
t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction)
}
if terminal.Value != test.wantValue {
t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue)
}
if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) {
t.Fatalf("rejected terminal = %#v, want rejection without value", terminal)
}
})
}
}
func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) {
calls := 0
policy := DefaultValidationPolicy()
policy.SemanticRejection = SemanticRejectionRejectOutput
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
calls++
return producerAttemptOutput{Value: "deterministic"}, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("deterministic_rejection"), nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 {
t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls)
}
}
func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
t.Run("retry", func(t *testing.T) {
calls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
calls++
if request.Number == 1 {
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
}
return producerAttemptOutput{Value: "replacement"}, nil
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 {
t.Fatalf("terminal = %#v, calls = %d", terminal, calls)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
})
t.Run("fallback", func(t *testing.T) {
fallbackWarning := contracts.Warning{ReasonCode: "fallback", Message: "fallback warning"}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackWarnings: []contracts.Warning{fallbackWarning}}}, nil
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Warnings, []contracts.Warning{fallbackWarning}) {
t.Fatalf("terminal = %#v", terminal)
}
})
}
func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) {
firstWarnings := []contracts.Warning{{ReasonCode: "discarded", Message: "discarded warning"}}
secondWarnings := []contracts.Warning{{ReasonCode: "accepted", Message: "accepted warning"}}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Warnings: firstWarnings}, nil
}
return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Warnings: secondWarnings}, nil
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "first" {
report := rejectedAttemptReport("defect")
report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")})
return report, nil
}
return validationReport{}, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted {
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
if got := terminal.Warnings; !reflect.DeepEqual(got, secondWarnings) {
t.Fatalf("terminal warnings = %#v, want %#v", got, secondWarnings)
}
if len(terminal.Provenance[0].Validation.records) != 2 {
t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records)
}
}
func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil
}
return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "first" {
return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil
}
return validationReport{}, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3)
if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) {
t.Fatalf("summary = %#v", summary)
}
if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 {
t.Fatalf("corrected success summary retained prior findings: %#v", summary)
}
if len(terminal.Warnings) != 0 {
t.Fatalf("corrected success warnings = %#v, want none", terminal.Warnings)
}
}
func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) {
report := validationReport{records: []validationRecord{
{validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")},
{validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"},
{validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")},
}}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "candidate"}, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
return report, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalIncompleteAccepted {
t.Fatalf("terminal action = %q", terminal.Action)
}
if got, want := terminal.Warnings, []contracts.Warning{
{Scope: "first", ReasonCode: "validator_execution_incomplete", Message: "Validator execution did not complete within its configured budget."},
{Scope: "third", ReasonCode: "validator_execution_incomplete", Message: "Validator execution did not complete within its configured budget."},
}; !reflect.DeepEqual(got, want) {
t.Fatalf("warnings = %#v, want %#v", got, want)
}
summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0)
if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) {
t.Fatalf("summary = %#v", summary)
}
}
func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) {
tests := []struct {
name string
context context.Context
produce producerAttemptProducer
}{
{
name: "cancelled context",
context: cancelledAttemptContext(),
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, nil
},
},
{
name: "debug persistence failure",
context: context.Background(),
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
calls := 0
producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
calls++
return test.produce(ctx, request)
}
terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt)
if err == nil || terminal.Action != producerTerminalFailed {
t.Fatalf("terminal = %#v, error = %v", terminal, err)
}
if test.name == "cancelled context" && calls != 0 {
t.Fatalf("cancelled producer calls = %d, want 0", calls)
}
if test.name == "debug persistence failure" && calls != 1 {
t.Fatalf("debug failure producer calls = %d, want 1", calls)
}
})
}
}
func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate {
t.Helper()
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
t.Fatalf("NewModelCandidate() error = %v", err)
}
return candidate
}
func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) {
return validationReport{}, nil
}
func rejectedAttemptReport(reason string) validationReport {
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}}
}
func failedAttemptReport() validationReport {
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}}
}
func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind {
kinds := make([]producerAttemptKind, len(provenance))
for index, item := range provenance {
kinds[index] = item.Kind
}
return kinds
}
func cancelledAttemptContext() context.Context {
ctx, cancel := context.WithCancel(context.Background())
cancel()
return ctx
}

View File

@@ -23,6 +23,8 @@ const (
type ModuleBinding struct { type ModuleBinding struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"` References map[string]ReferenceSource `json:"references,omitempty"`
@@ -82,6 +84,8 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
type moduleBindingJSON struct { type moduleBindingJSON struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"` References map[string]ReferenceSource `json:"references,omitempty"`
@@ -90,6 +94,8 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
out := moduleBindingJSON{ out := moduleBindingJSON{
Module: binding.Module, Module: binding.Module,
LLMProfile: binding.LLMProfile, LLMProfile: binding.LLMProfile,
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries, Retries: binding.Retries,
Options: binding.Options, Options: binding.Options,
References: binding.References, References: binding.References,
@@ -101,6 +107,44 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
return json.Marshal(out) return json.Marshal(out)
} }
func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
if binding == nil {
return fmt.Errorf("module binding must not be nil")
}
type moduleBindingJSON struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
Validators json.RawMessage `json:"validators,omitempty"`
}
var decoded moduleBindingJSON
if err := json.Unmarshal(data, &decoded); err != nil {
return err
}
out := ModuleBinding{
Module: decoded.Module,
LLMProfile: decoded.LLMProfile,
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
Retries: decoded.Retries,
Options: decoded.Options,
References: decoded.References,
}
if len(decoded.Validators) > 0 && string(decoded.Validators) != "null" {
var validators []ModuleBinding
if err := json.Unmarshal(decoded.Validators, &validators); err != nil {
return fmt.Errorf("decode module binding validators: %w", err)
}
out.Validators = ValidatorOverride{Set: true, Validators: validators}
}
*binding = cloneModuleBinding(out)
return nil
}
type ArtifactLaneProfile struct { type ArtifactLaneProfile struct {
Extract ModuleBinding `json:"extract"` Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"` Merge ModuleBinding `json:"merge,omitempty"`
@@ -118,6 +162,8 @@ type PipelineStepProfile struct {
type PipelineProfile struct { type PipelineProfile struct {
ID string `json:"id"` ID string `json:"id"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Input ModuleBinding `json:"input"` Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"` Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"` Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
@@ -167,10 +213,16 @@ type ResolvedArtifactLane struct {
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"` ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding Extract ModuleBinding
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"` ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
ExtractCorrectionProtocol contracts.CorrectionProtocol `json:"extract_correction_protocol,omitempty"`
ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
Merge ModuleBinding Merge ModuleBinding
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"` MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
MergeCorrectionProtocol contracts.CorrectionProtocol `json:"merge_correction_protocol,omitempty"`
MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
Normalize ModuleBinding Normalize ModuleBinding
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"` NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
NormalizeCorrectionProtocol contracts.CorrectionProtocol `json:"normalize_correction_protocol,omitempty"`
NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
Validators []ModuleBinding Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"` ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"` MergeReferences ResolvedReferenceTarget `json:"merge_references"`
@@ -199,10 +251,13 @@ type ResolvedValidator struct {
type ResolvedPipeline struct { type ResolvedPipeline struct {
ID string ID string
Digest string Digest string
ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
Input ModuleBinding Input ModuleBinding
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"` InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
Chunk ModuleBinding Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"` ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
ChunkCorrectionProtocol contracts.CorrectionProtocol `json:"chunk_correction_protocol,omitempty"`
ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"` ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
Steps []ResolvedPipelineStep Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"` ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
@@ -375,10 +430,12 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
} }
resolved := ResolvedPipeline{ resolved := ResolvedPipeline{
ID: pipelineID, ID: pipelineID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
Input: input, Input: input,
InputExecutionClass: inputModuleSpec.ExecutionClass, InputExecutionClass: inputModuleSpec.ExecutionClass,
Chunk: chunk, Chunk: chunk,
ChunkExecutionClass: chunkSpec.ExecutionClass, ChunkExecutionClass: chunkSpec.ExecutionClass,
ChunkCorrectionProtocol: chunkSpec.CorrectionProtocol,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences), ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: output, Output: output,
} }
@@ -450,6 +507,12 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil { if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
return ResolvedPipeline{}, err
}
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
return ResolvedPipeline{}, err
}
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil { if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -557,6 +620,7 @@ func resolveArtifactLane(
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references) lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
lane.ExtractReferences.StepID = strings.TrimSpace(stepID) lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
lane.ExtractExecutionClass = extractSpec.ExecutionClass lane.ExtractExecutionClass = extractSpec.ExecutionClass
lane.ExtractCorrectionProtocol = extractSpec.CorrectionProtocol
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType) mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
@@ -583,6 +647,7 @@ func resolveArtifactLane(
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences) lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
lane.MergeReferences.StepID = strings.TrimSpace(stepID) lane.MergeReferences.StepID = strings.TrimSpace(stepID)
lane.MergeExecutionClass = mergeSpec.ExecutionClass lane.MergeExecutionClass = mergeSpec.ExecutionClass
lane.MergeCorrectionProtocol = mergeSpec.CorrectionProtocol
capabilities.add(mergeSpec.Provides...) capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType) normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
@@ -609,6 +674,7 @@ func resolveArtifactLane(
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences) lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID) lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
lane.NormalizeCorrectionProtocol = normalizeSpec.CorrectionProtocol
capabilities.add(normalizeSpec.Provides...) capabilities.add(normalizeSpec.Provides...)
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {
@@ -870,6 +936,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
if err != nil { if err != nil {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q: %w", pipelineID, stage, chain.ModuleKey, err) return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q: %w", pipelineID, stage, chain.ModuleKey, err)
} }
if validator.Retries > 0 && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator %q retries require an LLM-backed execution class", pipelineID, stage, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{ chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator), Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass, ExecutionClass: spec.ExecutionClass,
@@ -1284,6 +1353,8 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
return ModuleBinding{ return ModuleBinding{
Module: module, Module: module,
LLMProfile: llmProfile, LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries, Retries: binding.Retries,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: references, References: references,
@@ -1359,6 +1430,125 @@ func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, over
return nil return nil
} }
func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pipelineAttempts *int) error {
apply := func(stage ModuleStage, laneID, module string, binding *ModuleBinding, executionClass contracts.ExecutionClass, kind string) error {
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
if executionClass != contracts.ExecutionClassLLMBacked {
if binding.StructuredOutputRepairAttempts != nil {
if laneID == "" {
return fmt.Errorf("pipeline %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, stage, module, kind, binding.Module)
}
return fmt.Errorf("pipeline %q lane %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, laneID, stage, module, kind, binding.Module)
}
return nil
}
if binding.StructuredOutputRepairAttempts == nil {
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(pipelineAttempts)
}
return nil
}
if err := apply(StageInput, "", resolved.Input.Module, &resolved.Input, resolved.InputExecutionClass, "module"); err != nil {
return err
}
if err := apply(StageChunk, "", resolved.Chunk.Module, &resolved.Chunk, resolved.ChunkExecutionClass, "module"); err != nil {
return err
}
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
if err := apply(StageExtract, lane.ID, lane.Extract.Module, &lane.Extract, lane.ExtractExecutionClass, "module"); err != nil {
return err
}
if err := apply(StageMerge, lane.ID, lane.Merge.Module, &lane.Merge, lane.MergeExecutionClass, "module"); err != nil {
return err
}
if err := apply(StageNormalize, lane.ID, lane.Normalize.Module, &lane.Normalize, lane.NormalizeExecutionClass, "module"); err != nil {
return err
}
}
}
if err := apply(StageOutput, "", resolved.Output.Module, &resolved.Output, resolved.OutputExecutionClass, "module"); err != nil {
return err
}
for chainIndex := range resolved.ValidatorChains {
chain := &resolved.ValidatorChains[chainIndex]
for validatorIndex := range chain.Validators {
validator := &chain.Validators[validatorIndex]
if err := apply(chain.Stage, chain.LaneID, chain.ModuleKey, &validator.Binding, validator.ExecutionClass, "validator"); err != nil {
return err
}
}
}
return nil
}
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
if binding.ValidationPolicy != nil {
if err := binding.ValidationPolicy.Validate(); err != nil {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
}
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
if laneID == "" {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
}
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
}
}
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
}
if pipelinePolicy != nil {
if err := pipelinePolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
}
}
if resolved.Input.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
}
if resolved.Output.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.Binding.ValidationPolicy != nil {
if chain.LaneID == "" {
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
}
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
}
}
}
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
if err != nil {
return err
}
resolved.ChunkValidationPolicy = policy
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
if err != nil {
return err
}
lane.ExtractValidationPolicy = policy
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
if err != nil {
return err
}
lane.MergeValidationPolicy = policy
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
if err != nil {
return err
}
lane.NormalizeValidationPolicy = policy
}
}
return nil
}
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) { func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
if len(bindings) == 0 { if len(bindings) == 0 {
return nil, nil return nil, nil
@@ -1453,10 +1643,13 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) { func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct { withoutDigest := struct {
ID string ID string
ConfiguredValidationPolicy *ValidationPolicyOverride
Input ModuleBinding Input ModuleBinding
InputExecutionClass contracts.ExecutionClass InputExecutionClass contracts.ExecutionClass
Chunk ModuleBinding Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass ChunkExecutionClass contracts.ExecutionClass
ChunkCorrectionProtocol contracts.CorrectionProtocol
ChunkValidationPolicy ValidationPolicy
ChunkReferences ResolvedReferenceTarget ChunkReferences ResolvedReferenceTarget
Steps []ResolvedPipelineStep Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain ValidatorChains []ResolvedValidatorChain
@@ -1464,10 +1657,13 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
OutputExecutionClass contracts.ExecutionClass OutputExecutionClass contracts.ExecutionClass
}{ }{
ID: resolved.ID, ID: resolved.ID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
ChunkValidationPolicy: resolved.ChunkValidationPolicy,
Input: resolved.Input, Input: resolved.Input,
InputExecutionClass: resolved.InputExecutionClass, InputExecutionClass: resolved.InputExecutionClass,
Chunk: resolved.Chunk, Chunk: resolved.Chunk,
ChunkExecutionClass: resolved.ChunkExecutionClass, ChunkExecutionClass: resolved.ChunkExecutionClass,
ChunkCorrectionProtocol: resolved.ChunkCorrectionProtocol,
ChunkReferences: resolved.ChunkReferences, ChunkReferences: resolved.ChunkReferences,
Steps: resolved.Steps, Steps: resolved.Steps,
ValidatorChains: resolved.ValidatorChains, ValidatorChains: resolved.ValidatorChains,

View File

@@ -126,6 +126,148 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) {
} }
} }
func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) {
correction := contracts.CorrectionProtocolSingleResponseV1
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.ChunkCorrectionProtocol != correction {
t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction {
t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction)
}
withoutCapability := cloneResolvedPipeline(resolved)
withoutCapability.ChunkCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = ""
withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability)
if err != nil {
t.Fatalf("resolvedPipelineDigest() error = %v", err)
}
if withoutCapabilityDigest == resolved.Digest {
t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if prepared.ChunkCorrectionProtocol != correction {
t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction)
}
preparedLane := prepared.Steps[0].ArtifactLanes[0]
if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction {
t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction)
}
}
func TestPrepareRequiresCorrectionCapabilityForValidatorRetries(t *testing.T) {
for _, test := range []struct {
name string
protocol contracts.CorrectionProtocol
retries int
validators bool
want string
}{
{name: "supported", protocol: contracts.CorrectionProtocolSingleResponseV1, retries: 1, validators: true},
{name: "unsupported", retries: 1, validators: true, want: "does not declare correction protocol"},
{name: "no validators", retries: 1},
{name: "no retries", validators: true},
} {
t.Run(test.name, func(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: test.protocol, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) {
return llmProfileTestChunkValidator{key: "chunk-validator"}, nil
}); err != nil {
t.Fatal(err)
}
profile := llmProfilePipeline()
profile.Chunk.Retries = test.retries
if test.validators {
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "chunk-validator"}}}
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
_, err = Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if test.want != "" {
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
}
return
}
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
})
}
}
type llmProfileTestChunkValidator struct{ key string }
func (validator llmProfileTestChunkValidator) Name() string { return validator.key }
func (llmProfileTestChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (llmProfileTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) {
for _, test := range []struct {
name string
class contracts.ExecutionClass
want string
}{
{name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"},
{name: "LLM validator", class: contracts.ExecutionClassLLMBacked},
} {
t.Run(test.name, func(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class})
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}}
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if test.want != "" {
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
}
return
}
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 {
t.Fatalf("validator retries = %d, want 1", got)
}
})
}
}
func TestResolvePipelineAppliesDefaults(t *testing.T) { func TestResolvePipelineAppliesDefaults(t *testing.T) {
resolved, err := ResolvePipeline(PipelineProfile{ resolved, err := ResolvePipeline(PipelineProfile{
ID: "defaulted", ID: "defaulted",
@@ -408,6 +550,148 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
} }
} }
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
pipelineAttempts := 2
chunkAttempts := 1
extractAttempts := 0
validatorAttempts := 3
profile := llmProfilePipeline()
profile.StructuredOutputRepairAttempts = &pipelineAttempts
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
lane := profile.Artifacts["events"]
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
encoded, err := json.Marshal(resolved)
if err != nil {
t.Fatalf("json.Marshal(resolved) error = %v", err)
}
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
}
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
for _, test := range []struct {
name string
got *int
want int
}{
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
} {
t.Run(test.name, func(t *testing.T) {
if test.got == nil || *test.got != test.want {
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
}
})
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
if extractChain == nil || len(extractChain.Validators) != 1 {
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
}
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
}
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
if chunkChain == nil || len(chunkChain.Validators) != 1 {
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
}
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
}
pipelineAttempts = 1
chunkAttempts = 2
extractAttempts = 3
validatorAttempts = 0
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
}
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
}
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
}
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
}
}
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*PipelineProfile)
}{
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
{name: "validator", mutate: func(profile *PipelineProfile) {
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
profile.Artifacts["events"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := baselineProfile()
test.mutate(&profile)
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
}
})
}
}
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
profile := baselineProfile()
profile.StructuredOutputRepairAttempts = repairAttempts(2)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
}
}
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
digests := make(map[string]string)
for _, test := range []struct {
name string
attempts *int
}{
{name: "prompt owned", attempts: nil},
{name: "disabled", attempts: repairAttempts(0)},
{name: "configured", attempts: repairAttempts(1)},
} {
t.Run(test.name, func(t *testing.T) {
profile := llmProfilePipeline()
profile.StructuredOutputRepairAttempts = test.attempts
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
digests[test.name] = resolved.Digest
})
}
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
}
}
func TestResolvePipelineRecordsValidatorChains(t *testing.T) { func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
catalog := newProfileCatalog(t) catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{ if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
@@ -1843,6 +2127,10 @@ func llmProfilePipeline() PipelineProfile {
} }
} }
func repairAttempts(value int) *int {
return &value
}
func llmProfileValues(profile string) map[string]string { func llmProfileValues(profile string) map[string]string {
return map[string]string{ return map[string]string{
"input": profile, "input": profile,

View File

@@ -65,6 +65,7 @@ type RunInput struct {
llmClient contracts.StructuredLLMClient llmClient contracts.StructuredLLMClient
stepID string stepID string
references map[referenceTargetKey]contracts.ReferenceSet references map[referenceTargetKey]contracts.ReferenceSet
referenceReuseEligibility map[referenceTargetKey]bool
} }
type RunOutput struct { type RunOutput struct {
@@ -75,6 +76,9 @@ type RunOutput struct {
Warnings []contracts.Warning `json:"warnings,omitempty"` Warnings []contracts.Warning `json:"warnings,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"` OutputFiles []contracts.OutputFile `json:"-"`
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"` CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
normalizeReuseEligibility map[generatedOutputKey]bool
} }
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) { func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
@@ -177,6 +181,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Path: input.Path, Path: input.Path,
Raw: input.RawInput, Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile, LLMProfile: input.pipeline.Input.LLMProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Input.StructuredOutputRepairAttempts),
Metadata: requestMetadata, Metadata: requestMetadata,
}) })
if ctxErr := ctx.Err(); ctxErr != nil { if ctxErr := ctx.Err(); ctxErr != nil {
@@ -240,6 +245,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if chunkResult.rejection != nil { if chunkResult.rejection != nil {
output.Rejected = append(output.Rejected, *chunkResult.rejection) output.Rejected = append(output.Rejected, *chunkResult.rejection)
} }
if chunkResult.validation != nil {
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(*chunkResult.validation))
}
output.Warnings = append(output.Warnings, chunkResult.warnings...) output.Warnings = append(output.Warnings, chunkResult.warnings...)
chunkDebugPayload := map[string]any{ chunkDebugPayload := map[string]any{
"cache_mode": chunkMode, "cache_mode": chunkMode,
@@ -296,6 +304,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if len(output.Rejected) > 0 { if len(output.Rejected) > 0 {
output.Manifest.ValidationStatus = "rejected" output.Manifest.ValidationStatus = "rejected"
} else if hasIncompleteValidation(output.ValidationSummaries) {
output.Manifest.ValidationStatus = "incomplete"
} else { } else {
output.Manifest.ValidationStatus = "approved" output.Manifest.ValidationStatus = "approved"
} }
@@ -365,6 +375,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Rejected: cloneRejectedOutputs(output.Rejected), Rejected: cloneRejectedOutputs(output.Rejected),
Warnings: output.Warnings, Warnings: output.Warnings,
LLMProfile: input.pipeline.Output.LLMProfile, LLMProfile: input.pipeline.Output.LLMProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Output.StructuredOutputRepairAttempts),
Metadata: outputMetadata, Metadata: outputMetadata,
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap), ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact), EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
@@ -399,6 +410,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil return output, nil
} }
func hasIncompleteValidation(summaries []artifacts.ValidationSummary) bool {
for _, summary := range summaries {
if summary.Status == "incomplete" {
return true
}
}
return false
}
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error { func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
for _, step := range input.Prepared.Steps { for _, step := range input.Prepared.Steps {
stepInput := input stepInput := input
@@ -408,6 +428,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err) return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
} }
stepInput.references = stepReferences stepInput.references = stepReferences
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
output.Manifest.References = append(output.Manifest.References, referenceProvenance...) output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks) laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
if err := mergeLaneOutput(output, laneOutput); err != nil { if err := mergeLaneOutput(output, laneOutput); err != nil {
@@ -426,7 +447,7 @@ type retryAttemptResult struct {
warnings []contracts.Warning warnings []contracts.Warning
} }
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (retryAttemptResult, error)) (retryAttemptResult, error) { func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (retryAttemptResult, error)) (retryAttemptResult, error) {
attempts := retries + 1 attempts := retries + 1
var last retryAttemptResult var last retryAttemptResult
for attempt := 1; attempt <= attempts; attempt++ { for attempt := 1; attempt <= attempts; attempt++ {
@@ -468,34 +489,36 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (retry
return last, nil return last, nil
} }
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) { func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
content, err := json.Marshal(chunks) content, err := json.Marshal(chunks)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err) return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err)
} }
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)} schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
var warnings []contracts.Warning report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
for index, item := range prepared.validators {
binding := item.resolved.Binding binding := item.resolved.Binding
started := time.Now().UTC() started := time.Now().UTC()
attemptPath := path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, fileio.EncodePathComponent(binding.Module), attempt)) attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
validatorCtx, llmScope := withIsolatedDebugLLMScope(ctx, attemptPath) validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
var result contracts.ValidationResult var result contracts.ValidationResult
requestMetadata, cloneErr := cloneMetadata(metadata) requestMetadata, cloneErr := cloneMetadata(metadata)
if cloneErr != nil { if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunk validation metadata: %w", cloneErr) return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
} }
requestChunks, cloneErr := cloneSourceChunks(chunks) requestChunks, cloneErr := cloneSourceChunks(chunks)
if cloneErr != nil { if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunks for validation: %w", cloneErr) return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
} }
switch item.resolved.Target { switch item.resolved.Target {
case ValidatorTargetChunk: case ValidatorTargetChunk:
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks}) result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
case ValidatorTargetSerialized: case ValidatorTargetSerialized:
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)}) result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
default: default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module) return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
}
if err == nil {
err = contracts.ValidateValidationResult(result)
} }
debugContent := debugContentEnvelope(content, "application/json", nil, nil) debugContent := debugContentEnvelope(content, "application/json", nil, nil)
debugContent.ContentDigest = debugContentDigest(content) debugContent.ContentDigest = debugContentDigest(content)
@@ -506,27 +529,19 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if err != nil { if err != nil {
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err) validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil { if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return warnings, nil, errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)) return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
} }
return warnings, nil, validationErr return validationInvocation{}, validationErr
} }
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil { if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
return warnings, nil, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr) return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
} }
if !result.Approved { return validationInvocation{result: result}, nil
reason := result.ReasonCode })
if reason == "" { if err != nil {
reason = "output_rejected" return report, err
} }
message := result.Message return report, nil
if message == "" {
message = "output rejected"
}
return warnings, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
}
warnings = append(warnings, result.Warnings...)
}
return warnings, nil, nil
} }
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain { func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
@@ -724,6 +739,9 @@ func populateOutputManifest(output *RunOutput) {
} }
output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs) output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected) output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
if len(output.ValidationSummaries) > 0 {
output.Manifest.ValidationSummaries = cloneValidationSummaries(output.ValidationSummaries)
}
if len(output.CheckpointEvents) > 0 { if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents)) decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents { for _, event := range output.CheckpointEvents {
@@ -773,6 +791,7 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
Message: output.Message, Message: output.Message,
AttemptCount: output.AttemptCount, AttemptCount: output.AttemptCount,
DiagnosticArtifactPath: output.DiagnosticArtifactPath, DiagnosticArtifactPath: output.DiagnosticArtifactPath,
Validation: cloneValidationSummaryPtr(output.Validation),
}) })
} }
return manifests return manifests
@@ -1031,7 +1050,31 @@ func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.Rejec
if len(rejected) == 0 { if len(rejected) == 0 {
return nil return nil
} }
return append([]contracts.RejectedOutput(nil), rejected...) cloned := make([]contracts.RejectedOutput, len(rejected))
for index, item := range rejected {
cloned[index] = item
cloned[index].Validation = cloneValidationSummaryPtr(item.Validation)
}
return cloned
}
func cloneValidationSummaryPtr(summary *artifacts.ValidationSummary) *artifacts.ValidationSummary {
if summary == nil {
return nil
}
cloned := artifacts.CloneValidationSummary(*summary)
return &cloned
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
} }
func timePtr(t time.Time) *time.Time { func timePtr(t time.Time) *time.Time {

View File

@@ -117,6 +117,38 @@ func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
return prepared return prepared
} }
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
producerAttempts := 2
validatorAttempts := 0
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
if len(lane.extractValidators.validators) == 0 {
t.Fatal("extract validators are empty")
}
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
var observedProducer, observedValidator *int
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
observedProducer = request.StructuredOutputRepairAttempts
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
})
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
observedValidator = target.structuredOutputRepairAttempts
return contracts.ValidationResult{Approved: true}, nil
}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
}
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
}
}
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error { func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil) _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
return err return err
@@ -198,14 +230,14 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
if attempts == 1 { if attempts == 1 {
scope = "discarded" scope = "discarded"
} }
return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Warnings: []contracts.Warning{{Scope: scope, ReasonCode: "observed", Message: scope}}}, nil return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Warnings: []contracts.Warning{{Scope: scope, ReasonCode: "observed", Message: scope}}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope))}, nil
} }
validatorCalls := 0 validatorCalls := 0
validator := preparedValidator{ validator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++ validatorCalls++
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate"}, nil return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
}, },
} }
switch stage { switch stage {
@@ -253,6 +285,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
path string path string
wantError string wantError string
wantBody string wantBody string
attemptError bool
}{ }{
{ {
name: "merge module error", name: "merge module error",
@@ -263,10 +296,12 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
}, },
path: "merge/notes/attempt-01.json", path: "merge/notes/attempt-01.json",
wantError: "merge exploded", wantError: "merge exploded",
attemptError: true,
}, },
{ {
name: "normalize validator error", name: "normalize validator error",
configure: func(prepared *PreparedPipeline) { configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureFailRun
prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{ prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
@@ -281,10 +316,11 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
{ {
name: "merge final rejection", name: "merge final rejection",
configure: func(prepared *PreparedPipeline) { configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{ prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
}, },
}} }}
}, },
@@ -300,6 +336,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
}, },
path: "normalize/notes/attempt-01.json", path: "normalize/notes/attempt-01.json",
wantError: "serialize normalize candidate", wantError: "serialize normalize candidate",
attemptError: true,
}, },
} }
@@ -311,9 +348,15 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
envelope := debug.envelope(t, tc.path) envelope := debug.envelope(t, tc.path)
if tc.wantError != "" { if tc.wantError != "" {
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) { if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError) t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
} }
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else if runErr != nil { } else if runErr != nil {
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr) t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
} }

View File

@@ -160,7 +160,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
validator := preparedValidator{ validator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator"}, nil return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator", CorrectionGuidance: "return an acceptable candidate"}, nil
}, },
} }
switch target { switch target {
@@ -187,6 +187,11 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) { t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t) prepared := preparedAttemptDebugPipeline(t)
if target == StageMerge {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
} else {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
}
prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{} prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{}
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{} prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{} prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
@@ -306,6 +311,11 @@ func TestRunnerRejectsCandidatesBeforeFinalEncoding(t *testing.T) {
for _, target := range []ModuleStage{StageMerge, StageNormalize} { for _, target := range []ModuleStage{StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) { t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t) prepared := preparedAttemptDebugPipeline(t)
if target == StageMerge {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
} else {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
}
codec := &observedNotesCodec{} codec := &observedNotesCodec{}
installObservedNotesCodec(t, prepared, codec) installObservedNotesCodec(t, prepared, codec)
configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}}) configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}})

Some files were not shown because too many files have changed in this diff Show More