Compare commits
45 Commits
ad1cba41c2
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e92bcfa74c | |||
| e6b2ae88d2 | |||
| b2e83bd6e7 | |||
| 7f01c3e79e | |||
| 7b5f4ebd42 | |||
| 4bb4582695 | |||
| 7d3434e5f6 | |||
| 29872b2e28 | |||
| b487d93186 | |||
| 8dd7a4324d | |||
| 04ba87e174 | |||
| a26d6ed042 | |||
| 759d32403f | |||
| 1a7b20c766 | |||
| 85a5b52be7 | |||
| 9d0faabf61 | |||
| 1c3da3e869 | |||
| 9abd93502f | |||
| c8a29a5fa2 | |||
| 916d9210fd | |||
| 3d3f16db4a | |||
| 63c397d86a | |||
| 9a92212632 | |||
| 0ef8931697 | |||
| e00cc45c6b | |||
| ab9b743df6 | |||
| 3bd3c7ebf7 | |||
| 75a3f51cee | |||
| 2be999ebd3 | |||
| 32fe7c5b98 | |||
| 55247c47ab | |||
| 5e5c69bf9d | |||
| b4a81f8b09 | |||
| adfed22e1a | |||
| 75b1e2f68b | |||
| 4473363d9f | |||
| 6eb45e0003 | |||
| 0585ad76dc | |||
| 56145c3b7e | |||
| a478fd86c5 | |||
| 916532100d | |||
| bef8ca263b | |||
| f6d037b613 | |||
| 449b506804 | |||
| 071a78ae22 |
33
.woodpecker/release.yml
Normal file
33
.woodpecker/release.yml
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
when:
|
||||||
|
- event: tag
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: validate-release
|
||||||
|
image: golang:1.25.5
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
version="$CI_COMMIT_TAG"
|
||||||
|
release_note="docs/releases/$version.md"
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
|
||||||
|
printf '%s\n' "invalid release tag: $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$release_note" ]; then
|
||||||
|
printf '%s\n' "missing release note: $release_note" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note heading does not match $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note is missing heading: $heading" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
./scripts/check-release-source.sh "$version"
|
||||||
16
README.md
16
README.md
@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
|
|||||||
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
||||||
demonstrates all implemented D&D lanes and the supporting campaign references.
|
demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||||
|
|
||||||
|
## Install A Source Release
|
||||||
|
|
||||||
|
Install a pinned source release with Go:
|
||||||
|
|
||||||
|
~~~
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
|
||||||
|
installed command's diagnostic version is described in the [CLI
|
||||||
|
reference](docs/cli.md); maintainers preparing a release should follow [Source
|
||||||
|
Releases](docs/release.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
||||||
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
|||||||
artifact formats.
|
artifact formats.
|
||||||
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
||||||
from an orchestrator and consume a published result.
|
from an orchestrator and consume a published result.
|
||||||
|
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
|
||||||
|
D&D pipeline as a subprocess and discover its structured artifacts.
|
||||||
- [Internal overview](docs/internal/overview.md) — implemented component map
|
- [Internal overview](docs/internal/overview.md) — implemented component map
|
||||||
for maintainers.
|
for maintainers.
|
||||||
- [Developer guide](docs/development.md) — contributor orientation and
|
- [Developer guide](docs/development.md) — contributor orientation and
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_combat_turns_llm.v1.json
|
schema_path: dnd_combat_turns_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -50,4 +50,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_enemy_events_llm.v1.json
|
schema_path: dnd_enemy_events_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_occurrences_llm.v1.json
|
schema_path: dnd_item_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_registry_llm.v1.json
|
schema_path: dnd_item_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -27,4 +27,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_occurrences_llm.v1.json
|
schema_path: dnd_location_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_registry_llm.v1.json
|
schema_path: dnd_location_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -27,4 +27,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_registry_llm.v1.json
|
schema_path: dnd_npc_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -27,4 +27,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -35,4 +35,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -31,4 +31,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scenes_llm.v1.json
|
schema_path: dnd_scenes_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -47,4 +47,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_spells_llm.v1.json
|
schema_path: dnd_spells_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -24,4 +24,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -83,9 +83,7 @@ entity identity, checkpoint contracts, or cross-request correlation. Changes
|
|||||||
to shared protocol and policy assets must participate in the normal prompt,
|
to shared protocol and policy assets must participate in the normal prompt,
|
||||||
schema, and checkpoint fingerprint mechanisms.
|
schema, and checkpoint fingerprint mechanisms.
|
||||||
|
|
||||||
Acceptance of this decision does not imply that the shared mechanism or its
|
The shared mechanism and its initial D&D registry consumers are now
|
||||||
consumer migrations are implemented. The
|
implemented. Current behavior is documented in
|
||||||
[feature roadmap](../roadmap/semantic-reconciliation.md) owns target behavior
|
[Module Internals](../internal/modules.md#semantic-reconciliation) and
|
||||||
and status, and the
|
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||||
[implementation plan](../roadmap/implementation.md) owns delivery sequence
|
|
||||||
until the work is complete.
|
|
||||||
|
|||||||
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# ADR-0014: Use feedback-aware validation retries
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-08-26
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Validation can identify a candidate defect after a producer has returned an
|
||||||
|
otherwise well-formed result. Retrying without the validator's deterministic,
|
||||||
|
bounded feedback wastes the useful diagnosis, while treating validator
|
||||||
|
execution failures as defects would ask a producer to repair conditions it
|
||||||
|
cannot control. The mechanism must preserve typed producer ownership,
|
||||||
|
checkpoint safety, and the repository's sensitive-data boundaries.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
The implementation will keep three independent budgets: the producer binding's
|
||||||
|
outer `retries` budget, PromptKit's structured-output repair budget, and each
|
||||||
|
validator's execution-retry budget. Validators will run sequentially in their
|
||||||
|
configured order and aggregate both rejections and execution failures before a
|
||||||
|
candidate disposition is selected.
|
||||||
|
|
||||||
|
A correction-capable producer will provide the exact single LLM response that
|
||||||
|
controlled its candidate using the `single_response_v1` protocol. A correction
|
||||||
|
attempt will reconstruct the ordinary request and append exactly two fresh
|
||||||
|
messages: that latest response as `assistant`, followed by one deterministic
|
||||||
|
aggregate correction request as `user`. Earlier turns will not accumulate.
|
||||||
|
|
||||||
|
Validator failures will not recurse into correction. Pipeline policy owns
|
||||||
|
terminal disposition, with field-by-field producer overrides over pipeline
|
||||||
|
defaults: structural failure and semantic rejection default to `fail_run`, and
|
||||||
|
validator execution failure defaults to `warn_continue`. Validators can report
|
||||||
|
facts and bounded corrective guidance, but never decide disposition.
|
||||||
|
|
||||||
|
Rejected and structurally invalid candidates will not advance. A candidate
|
||||||
|
allowed through after a validator execution failure will retain explicit
|
||||||
|
incomplete-validation provenance and will not be checkpointed. Exact response
|
||||||
|
and correction text remain attempt-local: they are excluded from ordinary
|
||||||
|
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
|
||||||
|
summaries.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Retry every producer after any validation outcome. This conflates producer
|
||||||
|
defects with validator operational failures and wastes retry budget.
|
||||||
|
- Let validators decide whether to continue. This would distribute pipeline
|
||||||
|
disposition policy across validators and undermine consistent defaults.
|
||||||
|
- Reuse the full prior conversation. Accumulated turns introduce unbounded
|
||||||
|
prompt growth and make correction behavior depend on incidental history.
|
||||||
|
- Persist raw responses to simplify diagnosis. Raw model output and correction
|
||||||
|
guidance may be sensitive and do not belong in durable pipeline records.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
The framework gains transport-neutral correction and candidate contracts,
|
||||||
|
producer capability checks, policy resolution, aggregated validation outcomes,
|
||||||
|
and conservative checkpoint handling. Prompt construction remains inside the
|
||||||
|
LLM adapter, while modules remain responsible for accurately exposing the
|
||||||
|
single response that directly controlled a candidate.
|
||||||
12
docs/cli.md
12
docs/cli.md
@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
|
|||||||
|
|
||||||
~~~
|
~~~
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||||
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
|
|||||||
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
||||||
writes the command summary to standard output and exits with status 0.
|
writes the command summary to standard output and exits with status 0.
|
||||||
|
|
||||||
|
`notarius --version` is valid only as the sole root argument. It writes exactly
|
||||||
|
`notarius <version>` followed by a newline to standard output and exits with
|
||||||
|
status 0. A tagged `go install` build can report its main-module stable tag,
|
||||||
|
and controlled builds can inject a stable tag at link time through
|
||||||
|
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
|
||||||
|
unversioned checkout reports `development`. Invalid injected version content is
|
||||||
|
a runtime error with exit status 1, while extra `--version` arguments are a
|
||||||
|
syntax error with exit status 2. This diagnostic does not replace the
|
||||||
|
[run-result](integrations/run-result.md) or artifact contracts for downstream
|
||||||
|
compatibility decisions.
|
||||||
|
|
||||||
## run
|
## run
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
|
|||||||
@@ -132,7 +132,11 @@ model: example-model
|
|||||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||||
its credential environment variable through `api_key_env`; set that variable
|
its credential environment variable through `api_key_env`; set that variable
|
||||||
only in the run environment. PromptKit owns the
|
only in the run environment. PromptKit owns the
|
||||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
|
||||||
|
including `base_profile` inheritance. Notarius passes profiles through without
|
||||||
|
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||||
|
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||||
|
provider without authorization.
|
||||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||||
supported package API, and [Operations](operations.md#operational-limits)
|
supported package API, and [Operations](operations.md#operational-limits)
|
||||||
describes the effective concurrency layers.
|
describes the effective concurrency layers.
|
||||||
@@ -218,6 +222,8 @@ pipelines:
|
|||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||||
|
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
|
||||||
|
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
|
||||||
| **input** | module binding | none | Required. |
|
| **input** | module binding | none | Required. |
|
||||||
| **chunk** | module binding | **generic** | Optional. |
|
| **chunk** | module binding | **generic** | Optional. |
|
||||||
| **output** | module binding | **json** | Optional. |
|
| **output** | module binding | **json** | Optional. |
|
||||||
@@ -237,6 +243,42 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
|
|||||||
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
||||||
Deterministic bindings do not receive these defaults or run overrides.
|
Deterministic bindings do not receive these defaults or run overrides.
|
||||||
|
|
||||||
|
Structural output repair is resolved after module, validator, and `--only` lane
|
||||||
|
selection. An object's **structured_output_repair_attempts** value takes
|
||||||
|
precedence over the pipeline value; otherwise, an LLM-backed binding or
|
||||||
|
validator inherits the pipeline value. If both are omitted, PromptKit uses the
|
||||||
|
prompt's declared repair policy. The value must be an integer from 0 through 3;
|
||||||
|
explicit `null` and non-integer values are invalid. An explicit value on a
|
||||||
|
deterministic binding or validator is invalid, while a pipeline value simply
|
||||||
|
does not apply to deterministic selections.
|
||||||
|
|
||||||
|
`validation_policy` controls terminal disposition for one complete producer
|
||||||
|
attempt and validator chain. It may appear on a pipeline or a **chunk**,
|
||||||
|
**extract**, **merge**, or **normalize** module binding; input, output, and
|
||||||
|
validator bindings reject it. Every field is optional and resolves in binding,
|
||||||
|
pipeline, then application-default order:
|
||||||
|
|
||||||
|
| Field | Values | Default |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
|
||||||
|
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
|
||||||
|
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
|
||||||
|
|
||||||
|
The policy object and its fields must be non-null, and unknown fields are
|
||||||
|
rejected. A deterministic producer may not explicitly set
|
||||||
|
**producer_structural_failure** on its binding, although a pipeline-level
|
||||||
|
default remains valid for pipelines that include LLM-backed producers.
|
||||||
|
|
||||||
|
After the producer binding's retry budget is exhausted, an invalid structured
|
||||||
|
response uses **producer_structural_failure**. One or more semantic validator
|
||||||
|
rejections use **semantic_rejection**; rejection takes precedence over an
|
||||||
|
exhausted validator failure or skip. With no rejection, an exhausted validator
|
||||||
|
failure or skip uses **validator_failure**. `reject_output` records the
|
||||||
|
terminal rejection without advancing that candidate. `warn_continue` is valid
|
||||||
|
only for validator execution failure: it advances a structurally valid,
|
||||||
|
otherwise unrejected result with incomplete-validation provenance and without
|
||||||
|
making it reusable checkpoint state.
|
||||||
|
|
||||||
A lane has these fields:
|
A lane has these fields:
|
||||||
|
|
||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
@@ -274,17 +316,32 @@ extract:
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
|
||||||
|
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
|
||||||
|
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
|
||||||
| **options** | object | none | Must satisfy the selected module. |
|
| **options** | object | none | Must satisfy the selected module. |
|
||||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||||
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||||
|
|
||||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||||
Validator bindings accept only **module**, **llm_profile**, and **options**.
|
Validator bindings accept only **module**, **llm_profile**,
|
||||||
They reject **references**, **retries**, and nested **validators**. Deterministic
|
**structured_output_repair_attempts**, **retries**, and **options**. Their
|
||||||
validators reject an explicit **llm_profile**. Deterministic module bindings
|
**retries** value is a non-negative additional validator-execution budget and
|
||||||
also reject an explicit **llm_profile**.
|
is valid only when the selected validator is LLM-backed. A validator retry
|
||||||
|
rechecks the same immutable candidate; it never regenerates the producer.
|
||||||
|
They reject
|
||||||
|
**validation_policy**, **references**, and nested **validators**. Deterministic
|
||||||
|
validators reject explicit **llm_profile** and
|
||||||
|
**structured_output_repair_attempts**.
|
||||||
|
Deterministic module bindings also reject those explicit fields.
|
||||||
|
|
||||||
|
An LLM-backed chunk, extract, merge, or normalize producer with both a
|
||||||
|
non-empty validator chain and positive **retries** must declare the supported
|
||||||
|
single-response correction capability. Preparation rejects a configuration
|
||||||
|
that could require semantic correction from a producer that cannot provide an
|
||||||
|
exact prior response. A deterministic producer, or an LLM attempt that did
|
||||||
|
not make a model call, cannot consume a semantic retry after rejection.
|
||||||
|
|
||||||
The **json** output module accepts optional **include_chunk_map** and
|
The **json** output module accepts optional **include_chunk_map** and
|
||||||
**evidence_context** settings:
|
**evidence_context** settings:
|
||||||
@@ -319,7 +376,8 @@ Unknown outer or nested option fields are rejected, as are incompatible YAML
|
|||||||
types. The allowlist remains valid when a run uses lane filtering: a configured
|
types. The allowlist remains valid when a run uses lane filtering: a configured
|
||||||
lane that is not active for that invocation simply contributes no evidence.
|
lane that is not active for that invocation simply contributes no evidence.
|
||||||
Evidence publication is opt-in because it can persist source text and metadata.
|
Evidence publication is opt-in because it can persist source text and metadata.
|
||||||
Its payload contract is [Published Evidence Context](integrations/evidence-context.md).
|
When enabled, it publishes the selected source-unit excerpt defined by the
|
||||||
|
[Published Evidence Context contract](integrations/evidence-context.md).
|
||||||
|
|
||||||
## References And Ordered Handoffs
|
## References And Ordered Handoffs
|
||||||
|
|
||||||
|
|||||||
208
docs/consumers/dnd-pipeline.md
Normal file
208
docs/consumers/dnd-pipeline.md
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
# Consuming The Complete D&D Pipeline
|
||||||
|
|
||||||
|
Use this workflow when an orchestrator runs the maintained complete D&D
|
||||||
|
pipeline and consumes its structured JSON artifacts. The generic
|
||||||
|
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
|
||||||
|
this guide connects that workflow to the complete D&D configuration, its
|
||||||
|
Seriatim input, and its artifact inventory.
|
||||||
|
|
||||||
|
The [CLI reference](../cli.md), [configuration reference](../config.md),
|
||||||
|
[run-result receipt](../integrations/run-result.md), and
|
||||||
|
[published JSON output contract](../integrations/json-output.md) remain the
|
||||||
|
canonical definitions of those public interfaces.
|
||||||
|
|
||||||
|
## Prepare And Validate The Deployment
|
||||||
|
|
||||||
|
Start from the maintained
|
||||||
|
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
|
||||||
|
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
|
||||||
|
artifact handoffs, campaign references, chunk-map publication, and evidence
|
||||||
|
context.
|
||||||
|
|
||||||
|
A deployment must provide its own PromptKit profile and campaign reference
|
||||||
|
files. Use absolute paths for service and subprocess deployments. In
|
||||||
|
particular, observe these different resolution rules:
|
||||||
|
|
||||||
|
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||||
|
file; and
|
||||||
|
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||||
|
directory.
|
||||||
|
|
||||||
|
Do not copy the repository example's relative profile path into a deployment
|
||||||
|
without also controlling that working directory. The complete path and profile
|
||||||
|
rules are defined in [Configuration](../config.md).
|
||||||
|
|
||||||
|
Preflight the deployed configuration before processing sessions and whenever
|
||||||
|
it changes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius config validate \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--pipeline dnd-session
|
||||||
|
```
|
||||||
|
|
||||||
|
Provide credentials through the environment or the documented configuration
|
||||||
|
mechanism. Do not put credentials in command arguments, generated
|
||||||
|
configuration, or logs.
|
||||||
|
|
||||||
|
## Supply The Transcript
|
||||||
|
|
||||||
|
The complete pipeline consumes a Seriatim JSON document. The
|
||||||
|
[Seriatim input contract](../integrations/seriatim.md) defines its required
|
||||||
|
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
|
||||||
|
citations use those segment IDs as source-unit ranges.
|
||||||
|
|
||||||
|
When the caller maintains several transcript tiers, use the final trimmed JSON
|
||||||
|
transcript so extraction operates on the same session content presented to
|
||||||
|
later consumers. For example, Narratio identifies this implemented artifact as
|
||||||
|
`narratio.transcript.final_trimmed` and normally stores it at
|
||||||
|
`transcripts/final.trimmed.json`.
|
||||||
|
|
||||||
|
Notarius generates a stable prompt session from the resolved input module and
|
||||||
|
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
|
||||||
|
Use that override only when intentionally changing the routing relationship
|
||||||
|
between invocations; it is not a credential or output identity.
|
||||||
|
|
||||||
|
## Run Notarius
|
||||||
|
|
||||||
|
Invoke the pipeline with explicit absolute paths and request its
|
||||||
|
machine-readable receipt:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius run dnd-session \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||||
|
--output-dir /absolute/path/to/notarius-output \
|
||||||
|
--json
|
||||||
|
```
|
||||||
|
|
||||||
|
The caller should:
|
||||||
|
|
||||||
|
- capture stdout and stderr separately;
|
||||||
|
- propagate cancellation and impose an operator-appropriate timeout;
|
||||||
|
- wait for process completion before interpreting stdout; and
|
||||||
|
- retain stderr for diagnosis without copying secrets or transcript content
|
||||||
|
into other logs.
|
||||||
|
|
||||||
|
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
|
||||||
|
nonzero exit because a failed receipt write can leave partial bytes. The
|
||||||
|
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
|
||||||
|
stream and exit-status contract.
|
||||||
|
|
||||||
|
## Discover The Published Bundle
|
||||||
|
|
||||||
|
Decode the successful stdout document as a supported run-result schema. For
|
||||||
|
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate
|
||||||
|
unknown fields allowed by that version, but reject an unsupported schema
|
||||||
|
version.
|
||||||
|
|
||||||
|
Use the receipt's absolute `output_directory` as the exact run-specific bundle
|
||||||
|
root. Do not scan the output root for its newest directory, guess a run ID, or
|
||||||
|
construct a bundle path. Resolve `index_file` beneath `output_directory` and
|
||||||
|
reject an absolute logical path or any result that escapes the bundle root.
|
||||||
|
|
||||||
|
The complete configuration uses the application validation defaults. A caller
|
||||||
|
that requires fully validated D&D artifacts must also require receipt
|
||||||
|
`validation_status: approved`; a successful `incomplete` result reflects the
|
||||||
|
configured validator-failure continuation policy and carries its bounded
|
||||||
|
validator provenance in `validation_summaries`.
|
||||||
|
|
||||||
|
Read `index.json` and locate each requested lane in `output_files` by its exact
|
||||||
|
`lane_id`. Do not guess a lane filename. Before decoding a payload:
|
||||||
|
|
||||||
|
1. resolve its descriptor's relative `file` beneath the bundle root with the
|
||||||
|
same confinement check;
|
||||||
|
2. verify the descriptor's media type and schema identity against the linked
|
||||||
|
artifact contract; and
|
||||||
|
3. decode the payload according to that contract.
|
||||||
|
|
||||||
|
The [published JSON output contract](../integrations/json-output.md) defines
|
||||||
|
the index and bundle layout. Treat all paths obtained from a decoded external
|
||||||
|
document as untrusted until confined to their documented root.
|
||||||
|
|
||||||
|
## Complete Artifact Inventory
|
||||||
|
|
||||||
|
When every configured lane is accepted, the complete example publishes these
|
||||||
|
lane artifacts:
|
||||||
|
|
||||||
|
| Lane ID | Purpose | Canonical contract |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
|
||||||
|
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
|
||||||
|
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
|
||||||
|
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
|
||||||
|
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
|
||||||
|
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
|
||||||
|
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
|
||||||
|
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
|
||||||
|
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
|
||||||
|
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
|
||||||
|
|
||||||
|
The JSON encoder always publishes these bundle-management files:
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
|
||||||
|
| `manifest.json` | Run provenance and result summaries. |
|
||||||
|
| `rejected.json` | Rejected pipeline outputs. |
|
||||||
|
| `warnings.json` | Accepted-output and run warnings. |
|
||||||
|
|
||||||
|
The complete configuration also requests two pipeline-wide artifacts:
|
||||||
|
|
||||||
|
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
|
||||||
|
chunk metadata; and
|
||||||
|
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
|
||||||
|
excerpt containing the union of selected cited source units and the
|
||||||
|
configured surrounding window.
|
||||||
|
|
||||||
|
Discover both from their top-level `index.json` descriptors rather than
|
||||||
|
treating them as lanes. Evidence context is convenient reading material, not
|
||||||
|
authoritative provenance; citations in the normalized lane payloads remain the
|
||||||
|
evidence contract.
|
||||||
|
|
||||||
|
Every optional or lane file is published only when its corresponding artifact
|
||||||
|
is available. A successful process does not guarantee that all configured
|
||||||
|
lanes were accepted.
|
||||||
|
|
||||||
|
## Decide What Counts As Consumer Success
|
||||||
|
|
||||||
|
Exit status 0 means Notarius completed the pipeline and published its result
|
||||||
|
bundle. The receipt or bundle may still report warnings, rejected outputs, or
|
||||||
|
missing lane descriptors. A downstream consumer must define its own required
|
||||||
|
artifact set explicitly.
|
||||||
|
|
||||||
|
A caller that claims to consume the complete D&D workflow should normally
|
||||||
|
require all ten lane IDs in the table and verify each descriptor's expected
|
||||||
|
contract. If any required lane is missing, rejected, or incompatible, fail the
|
||||||
|
caller's extraction step while retaining the Notarius bundle for diagnosis. A
|
||||||
|
consumer that needs only a subset may define and document a narrower policy.
|
||||||
|
|
||||||
|
Keep the successful receipt with the complete published bundle. Retain
|
||||||
|
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
|
||||||
|
required by the caller's provenance, diagnosis, and retention policies. Avoid
|
||||||
|
selectively copying payload files without also preserving enough index and
|
||||||
|
manifest information to identify their originating run and contracts.
|
||||||
|
|
||||||
|
The transcript, lane artifacts, evidence context, manifest, debug data, and
|
||||||
|
logs can all contain private campaign information. Apply the same access,
|
||||||
|
publication, and retention controls used for the source transcript.
|
||||||
|
|
||||||
|
## Consumer Checklist
|
||||||
|
|
||||||
|
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
|
||||||
|
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
|
||||||
|
- Use absolute configuration, input, output-root, profile, and reference paths
|
||||||
|
in service deployments.
|
||||||
|
- Capture stdout and stderr separately and enforce cancellation and timeout.
|
||||||
|
- Parse stdout only after exit status 0.
|
||||||
|
- Accept only supported receipt, index, and artifact schema versions while
|
||||||
|
tolerating permitted unknown fields.
|
||||||
|
- Use the receipt's `output_directory`; never guess the run directory.
|
||||||
|
- Confine `index_file` and every descriptor path to the published bundle root.
|
||||||
|
- Discover lanes by `lane_id` and verify descriptor compatibility before
|
||||||
|
decoding payloads.
|
||||||
|
- Enforce an explicit required-lane policy and inspect rejections and warnings.
|
||||||
|
- Preserve the receipt and sufficient bundle provenance for every retained
|
||||||
|
artifact.
|
||||||
|
- Protect all transcript-derived files and diagnostic streams as sensitive
|
||||||
|
campaign data.
|
||||||
@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
|
|||||||
[Published JSON Output contract](../integrations/json-output.md) own the
|
[Published JSON Output contract](../integrations/json-output.md) own the
|
||||||
durable result formats.
|
durable result formats.
|
||||||
|
|
||||||
|
For the maintained complete D&D workflow, including its transcript input,
|
||||||
|
configured lane inventory, and downstream acceptance checklist, see
|
||||||
|
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
|
||||||
|
|
||||||
## Run And Check The Process
|
## Run And Check The Process
|
||||||
|
|
||||||
Optionally preflight a selected configuration and pipeline before work starts:
|
Optionally preflight a selected configuration and pipeline before work starts:
|
||||||
@@ -55,16 +59,23 @@ contract. The JSON bundle contract links to the available lane contracts.
|
|||||||
If `index.json` has an `evidence_context` descriptor, treat it as a
|
If `index.json` has an `evidence_context` descriptor, treat it as a
|
||||||
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
||||||
fields before decoding the linked file according to the [Published Evidence
|
fields before decoding the linked file according to the [Published Evidence
|
||||||
Context contract](../integrations/evidence-context.md). Use each
|
Context contract](../integrations/evidence-context.md). Decode its top-level
|
||||||
`evidence_refs` entry as the citation to source material. Its surrounding
|
source-unit array as a reading excerpt. Obtain authoritative citations and lane
|
||||||
context range and included units explain the citation, but do not widen or
|
provenance from the normalized lane artifacts; the excerpt has neither and its
|
||||||
replace the cited source reference.
|
nearby units do not widen a lane artifact's cited source reference.
|
||||||
|
|
||||||
A zero exit status may still report rejected outputs, warnings, or absent
|
A zero exit status may still report rejected outputs, warnings, or absent
|
||||||
lanes. The caller decides which lane IDs are required for its own work and
|
lanes. The caller decides which lane IDs are required for its own work and
|
||||||
which are optional; it should make that decision explicitly rather than infer
|
which are optional; it should make that decision explicitly rather than infer
|
||||||
failure from the receipt counts alone.
|
failure from the receipt counts alone.
|
||||||
|
|
||||||
|
When complete validation is required, also require receipt
|
||||||
|
`validation_status: approved` and inspect `validation_summaries`. A successful
|
||||||
|
run with `validation_status: incomplete` contains a structurally valid result
|
||||||
|
that advanced after validator execution could not complete under the configured
|
||||||
|
`warn_continue` policy. It is not reusable checkpoint state and should not be
|
||||||
|
silently treated as fully reviewed by the caller.
|
||||||
|
|
||||||
## Preserve Provenance And Handle Data Carefully
|
## Preserve Provenance And Handle Data Carefully
|
||||||
|
|
||||||
Keep the receipt with the published `manifest.json`, and retain
|
Keep the receipt with the published `manifest.json`, and retain
|
||||||
@@ -73,5 +84,5 @@ them. Treat the input, output bundle, cache, debug bundle, and captured process
|
|||||||
logs as potentially sensitive data. Apply the caller's access controls and
|
logs as potentially sensitive data. Apply the caller's access controls and
|
||||||
retention policy, and avoid copying secrets into arguments, logs, or
|
retention policy, and avoid copying secrets into arguments, logs, or
|
||||||
provenance records. An evidence-context artifact contains source-unit text and
|
provenance records. An evidence-context artifact contains source-unit text and
|
||||||
metadata, and selected lanes can cover most of an input; preserve and share it
|
metadata and can cover most of an input; preserve and share it only when that
|
||||||
only when that source content is authorized for the recipient.
|
source content is authorized for the recipient.
|
||||||
|
|||||||
@@ -18,13 +18,14 @@ implemented component map.
|
|||||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||||
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
||||||
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
||||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
|
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
|
||||||
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
||||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||||
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
||||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
||||||
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||||
|
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
|
||||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||||
|
|
||||||
For an existing subsystem, also inspect its focused tests and the package-local
|
For an existing subsystem, also inspect its focused tests and the package-local
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# Published Evidence Context
|
# Published Evidence Context
|
||||||
|
|
||||||
This contract defines the optional `source/evidence-context` artifact emitted
|
This contract defines the optional `source/evidence-context` artifact emitted
|
||||||
by the production JSON output. Its configuration is owned by
|
by the production JSON output. It is a selected source-unit excerpt for
|
||||||
[Configuration](../config.md#module-bindings-and-validators); its logical-file
|
convenient reading alongside normalized lane artifacts; it is not a second
|
||||||
discovery is owned by [Published JSON Output](json-output.md).
|
citation or provenance model. Its configuration is owned by
|
||||||
|
[Configuration](../config.md#module-bindings-and-validators), and its
|
||||||
|
logical-file discovery is owned by [Published JSON Output](json-output.md).
|
||||||
|
|
||||||
## Identity And Discovery
|
## Identity And Discovery
|
||||||
|
|
||||||
@@ -26,35 +28,13 @@ its absence means evidence publication was not enabled for that bundle.
|
|||||||
|
|
||||||
## Payload
|
## Payload
|
||||||
|
|
||||||
The v1 payload is a JSON object with required `source_id`, `source_digest`,
|
The v1 payload is a top-level JSON array of generic source units. There is no
|
||||||
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
|
wrapper, source-level metadata, context grouping, lane identifier, or evidence
|
||||||
`contexts` are always arrays; an enabled configuration with no accepted direct
|
reference in the payload. An enabled configuration with no contributing
|
||||||
evidence publishes `contexts: []`.
|
accepted evidence publishes `[]`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
[
|
||||||
"source_id": "session-alpha",
|
|
||||||
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
|
||||||
"window_units": 1,
|
|
||||||
"selected_lanes": ["npc_registry", "spells"],
|
|
||||||
"contexts": [
|
|
||||||
{
|
|
||||||
"context_ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 20
|
|
||||||
},
|
|
||||||
"evidence_refs": [
|
|
||||||
{
|
|
||||||
"lane_id": "spells",
|
|
||||||
"source_ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"units": [
|
|
||||||
{
|
{
|
||||||
"id": 10,
|
"id": 10,
|
||||||
"kind": "transcript_segment",
|
"kind": "transcript_segment",
|
||||||
@@ -76,41 +56,52 @@ evidence publishes `contexts: []`.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Each context requires a `context_ref` object and `evidence_refs` and `units`
|
Each source unit has required `id`, `kind`, `text`, and self `ref` fields.
|
||||||
arrays. `context_ref` identifies the first and last included unit. Each
|
`ref` contains `source_id`, `start_unit_id`, and `end_unit_id`, and both unit
|
||||||
evidence entry contains a selected `lane_id` and an original `source_ref`. A
|
endpoints identify that unit's `id`. A unit may also contain source-owned
|
||||||
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
|
`metadata`, an open-ended JSON object. Fixed unit and reference fields are
|
||||||
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
|
strict: consumers must reject unknown fixed fields, malformed units, invalid
|
||||||
unknown fields; unit metadata may contain application-defined JSON values.
|
self-references, units whose `source_id` differs from other units in the same
|
||||||
|
excerpt, and a payload that is not the array described here.
|
||||||
|
|
||||||
## Citations And Context
|
The excerpt preserves each selected unit exactly as represented by the
|
||||||
|
validated generic source document. It does not add evidence-context-specific
|
||||||
|
annotations or reshape source-owned metadata.
|
||||||
|
|
||||||
`evidence_refs` are the authoritative citations. They identify the direct
|
## Selection And Citations
|
||||||
references emitted by accepted normalized artifacts. `context_ref` and the
|
|
||||||
units collection include those cited units plus nearby source units selected by
|
|
||||||
the configured window. They are explanatory context, not widened citations.
|
|
||||||
|
|
||||||
Only accepted outputs from the configured lane allowlist contribute. Rejected,
|
The framework obtains direct source references only through typed evidence
|
||||||
failed, absent, and lane-filtered outputs do not contribute. The artifact never
|
projections of accepted normalized artifacts in the configured lane allowlist.
|
||||||
contains raw input bytes, prompts, model responses, auxiliary reference
|
It validates each reference against the current source document, expands its
|
||||||
content, credentials, or filesystem paths.
|
range by `window_units` source-unit positions on each side, clamps at document
|
||||||
|
boundaries, and takes the union of all expanded ranges. The output contains
|
||||||
|
each selected source unit once in source-document position order, regardless
|
||||||
|
of numeric unit IDs. Repeated references, overlapping windows, and citations
|
||||||
|
from multiple lanes do not duplicate a unit. Rejected, failed, absent,
|
||||||
|
inactive, and unselected lanes contribute nothing.
|
||||||
|
|
||||||
## Ordering And Compatibility
|
Normalized lane artifacts remain authoritative for citations and for which lane
|
||||||
|
cited a range. The excerpt has no lane attribution and must not be used to
|
||||||
|
reconstruct it. Its included nearby units provide reading context only; they
|
||||||
|
do not widen any citation in a lane artifact.
|
||||||
|
|
||||||
The selected lane allowlist is lexical. Contexts and units are in source
|
The excerpt contains at most every generic source unit once. It can therefore
|
||||||
document position order, not numeric unit-ID order. Direct evidence entries
|
equal the complete generic source document when coverage is broad or the
|
||||||
are deterministically ordered by lane and source reference. Overlapping or
|
window is large. No byte-, token-, or compression-size guarantee is made, and
|
||||||
contiguous windows merge, and each source unit appears at most once in the
|
the framework does not truncate the excerpt to meet an arbitrary size limit.
|
||||||
resulting contexts.
|
|
||||||
|
## Consumer Responsibilities And Data Handling
|
||||||
|
|
||||||
The artifact is additive to the JSON bundle and is not a lane payload,
|
The artifact is additive to the JSON bundle and is not a lane payload,
|
||||||
normalized-output count, checkpoint, or generated reference. Consumers that
|
normalized-output count, checkpoint, or generated reference. Consumers that
|
||||||
do not need it must tolerate the absent optional descriptor. Consumers that do
|
do not need it must tolerate an absent descriptor. Consumers that do use it
|
||||||
use it should preserve the artifact and its schema identity with the run
|
should validate the descriptor and payload before use, retain the artifact with
|
||||||
provenance, and should treat its source text and metadata as sensitive durable
|
its schema identity when needed for a run record, and read citations from the
|
||||||
content.
|
corresponding normalized lane artifacts.
|
||||||
|
|
||||||
|
The excerpt contains source-unit text and source-owned metadata and is durable
|
||||||
|
output. Treat it as sensitive source content, apply appropriate access controls
|
||||||
|
and retention, and do not assume its selected form is materially smaller or
|
||||||
|
less sensitive than the original input.
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ root for the logical discovery described here.
|
|||||||
| `warnings.json` | Accepted-output and run warnings. |
|
| `warnings.json` | Accepted-output and run warnings. |
|
||||||
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
||||||
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
||||||
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
|
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
|
||||||
|
|
||||||
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
||||||
accepted only when their media type is `application/json`.
|
accepted only when their media type is `application/json`.
|
||||||
@@ -92,7 +92,7 @@ group into the following externally observable summaries:
|
|||||||
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
||||||
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
||||||
| Source and references | `source_digests`, `references` |
|
| Source and references | `source_digests`, `references` |
|
||||||
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
|
| Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
|
||||||
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
||||||
|
|
||||||
`references` records provenance such as the target, slot, origin, digest,
|
`references` records provenance such as the target, slot, origin, digest,
|
||||||
@@ -102,6 +102,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
|
|||||||
provenance for the plan used by this run; cache records, debug artifacts, and
|
provenance for the plan used by this run; cache records, debug artifacts, and
|
||||||
other operational state are not published as bundle files.
|
other operational state are not published as bundle files.
|
||||||
|
|
||||||
|
Each `validation_summaries` entry is a bounded outcome for one producer result.
|
||||||
|
It has required `status`, `producer_attempt_count`, and `terminal_action`;
|
||||||
|
the stage and affected step, lane, module, or chunk identity are present when
|
||||||
|
applicable. `status` is `complete`, `rejected`, or `incomplete`.
|
||||||
|
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
|
||||||
|
configured validator order and omit later duplicates. Entries contain no raw
|
||||||
|
candidate response, correction guidance, validator diagnostic message, or
|
||||||
|
artifact payload. The same shape may appear as `validation` on an affected
|
||||||
|
rejection entry.
|
||||||
|
|
||||||
When present, `metadata.session_id` is the effective non-secret routing
|
When present, `metadata.session_id` is the effective non-secret routing
|
||||||
correlation identifier used for the run. It can be visible to providers and is
|
correlation identifier used for the run. It can be visible to providers and is
|
||||||
not a substitute for a cache or checkpoint identity. Its generation and
|
not a substitute for a cache or checkpoint identity. Its generation and
|
||||||
@@ -127,7 +137,10 @@ distinct even when their profile, provider, and model are otherwise equal.
|
|||||||
`rejected.json` is always an object with a `rejected` array. Each entry has
|
`rejected.json` is always an object with a `rejected` array. Each entry has
|
||||||
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
||||||
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
||||||
`diagnostic_artifact_path` are present only when applicable.
|
`diagnostic_artifact_path` are present only when applicable. An entry may also
|
||||||
|
contain the bounded `validation` summary described above; the existing singular
|
||||||
|
validator and reason fields remain the first configured rejection for
|
||||||
|
compatibility.
|
||||||
|
|
||||||
`warnings.json` is always an object with a `warnings` array. Each warning has
|
`warnings.json` is always an object with a `warnings` array. Each warning has
|
||||||
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
|
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# PromptKit Integration
|
# PromptKit Integration
|
||||||
|
|
||||||
Notarius pins
|
Notarius pins
|
||||||
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
|
[`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
|
||||||
as its in-process prompt engine. The upstream
|
as its in-process prompt engine. The upstream
|
||||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
|
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
owns the public engine API, and the upstream
|
owns the public engine API, and the upstream
|
||||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
owns prompt, profile, and schema file contracts.
|
owns prompt, profile, and schema file contracts.
|
||||||
|
|
||||||
## Supported Boundary
|
## Supported Boundary
|
||||||
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
||||||
operator and application-fallback profile sources;
|
operator and application-fallback profile sources;
|
||||||
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
||||||
variables, a direct session ID, prompt identity, and profile selection, then
|
variables, a direct session ID, prompt identity, profile selection, and
|
||||||
|
optional appended rendered messages, then
|
||||||
record credential-redacted details and run that exact execution;
|
record credential-redacted details and run that exact execution;
|
||||||
- return rendered debug material, validated structured output, selected
|
- return rendered debug material, validated structured output, selected
|
||||||
profile, backend, effective model metadata, and token usage;
|
profile, backend, effective model metadata, and token usage;
|
||||||
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
admission exhaustion through `ErrCapacityExceeded`.
|
admission exhaustion through `ErrCapacityExceeded`.
|
||||||
|
|
||||||
The pinned
|
The pinned
|
||||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
|
||||||
owns the registration and backend-capacity contract.
|
owns the registration and backend-capacity contract.
|
||||||
|
|
||||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||||
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
|
|||||||
field, which is authoritative for provider session behavior. It also retains
|
field, which is authoritative for provider session behavior. It also retains
|
||||||
the same value as the `session_id` prompt variable for maintained prompt
|
the same value as the `session_id` prompt variable for maintained prompt
|
||||||
compatibility. The generated identifier is 76 ASCII characters, within
|
compatibility. The generated identifier is 76 ASCII characters, within
|
||||||
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret
|
PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
|
||||||
correlation identifiers and may be exposed to providers and provider
|
correlation identifiers and may be exposed to providers and provider
|
||||||
observability. The CLI contract owns generation and override behavior.
|
observability. The CLI contract owns generation and override behavior.
|
||||||
|
|
||||||
@@ -84,7 +85,39 @@ configuration and deployment workflow are defined in
|
|||||||
[Configuration](../config.md#promptkit-profiles) and
|
[Configuration](../config.md#promptkit-profiles) and
|
||||||
[Operations](../operations.md#promptkit-profile-deployment).
|
[Operations](../operations.md#promptkit-profile-deployment).
|
||||||
|
|
||||||
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
|
PromptKit owns `base_profile` resolution under its
|
||||||
|
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
|
Notarius records the selected leaf identity and resolved target without parsing
|
||||||
|
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||||
|
reach the provider without authorization, which can result in a 401 or 403.
|
||||||
|
|
||||||
|
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
|
||||||
|
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
||||||
|
Maintained Notarius prompt definitions use only `system` and `user`.
|
||||||
|
|
||||||
|
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
|
||||||
|
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
|
||||||
|
exactly two messages in order: the latest validated producer response with
|
||||||
|
role `assistant`, then deterministic validation guidance with role `user`.
|
||||||
|
It never exposes a general caller-selected role API, accumulates earlier
|
||||||
|
correction turns, or changes the ordinary prompt prefix. Ordinary requests
|
||||||
|
leave appended messages unset.
|
||||||
|
|
||||||
|
PromptKit preserves supplied content but does not own Notarius's correction
|
||||||
|
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
|
||||||
|
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
|
||||||
|
1,114,112 bytes) before preparing the request. The transport-neutral
|
||||||
|
application contract owns defensive copying and these limits. Default request
|
||||||
|
and terminal summaries retain only safe counts, digests, identities, and usage;
|
||||||
|
complete appended messages remain limited to the explicitly requested detailed
|
||||||
|
debug trace.
|
||||||
|
|
||||||
|
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
|
||||||
|
profile catalogs from independently versioned transitive modules. Notarius
|
||||||
|
does not import or register either catalog; PromptKit retains catalog source,
|
||||||
|
identity, precedence, credential, and capacity ownership.
|
||||||
|
|
||||||
|
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
|
||||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||||
@@ -97,6 +130,10 @@ pinned upstream documentation.
|
|||||||
module assets, maps its transport-neutral completion contract, prepares and
|
module assets, maps its transport-neutral completion contract, prepares and
|
||||||
executes requests, validates output, records provenance, captures debug
|
executes requests, validates output, records provenance, captures debug
|
||||||
material, redacts errors, and preserves timeout ownership.
|
material, redacts errors, and preserves timeout ownership.
|
||||||
|
|
||||||
|
PromptKit provider error details do not cross the ordinary completion boundary.
|
||||||
|
Notarius exposes a provider-neutral generation category and optional status;
|
||||||
|
redacted provider details are retained only in requested debug material.
|
||||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||||
@@ -105,4 +142,7 @@ the conventional local backend.
|
|||||||
|
|
||||||
PromptKit API or format changes outside this boundary are not implicitly
|
PromptKit API or format changes outside this boundary are not implicitly
|
||||||
supported. Updating the pinned version requires reviewing the adapter and
|
supported. Updating the pinned version requires reviewing the adapter and
|
||||||
profile/configuration contracts against the upstream documentation.
|
profile/configuration contracts against the upstream documentation. Maintained
|
||||||
|
production prompts use PromptKit's bounded structural-repair contract; their
|
||||||
|
current declaration is one additional repair attempt. Notarius retains the
|
||||||
|
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||||
|
|||||||
@@ -22,11 +22,15 @@ The current schema version is `notarius.run-result.v1`.
|
|||||||
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
||||||
| `warning_count` | Yes | Number of final run warnings. |
|
| `warning_count` | Yes | Number of final run warnings. |
|
||||||
| `validation_status` | Yes | The final run manifest validation status. |
|
| `validation_status` | Yes | The final run manifest validation status. |
|
||||||
|
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
|
||||||
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
||||||
|
|
||||||
For the production `json` output module, `index_file` is present only when the
|
For the production `json` output module, `index_file` is present only when the
|
||||||
completed run returned exactly one logical output file named `index.json`.
|
completed run returned exactly one logical output file named `index.json`.
|
||||||
For another output module, its absence does not indicate a failed run.
|
For another output module, its absence does not indicate a failed run.
|
||||||
|
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
|
||||||
|
means one or more otherwise accepted results advanced under validator-failure
|
||||||
|
`warn_continue`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -38,7 +42,17 @@ For another output module, its absence does not indicate a failed run.
|
|||||||
"normalized_output_count": 6,
|
"normalized_output_count": 6,
|
||||||
"rejected_output_count": 2,
|
"rejected_output_count": 2,
|
||||||
"warning_count": 1,
|
"warning_count": 1,
|
||||||
"validation_status": "rejected"
|
"validation_status": "incomplete",
|
||||||
|
"validation_summaries": [
|
||||||
|
{
|
||||||
|
"stage": "extract",
|
||||||
|
"lane_id": "spells",
|
||||||
|
"status": "incomplete",
|
||||||
|
"incomplete_validators": ["dnd/spells/source_refs"],
|
||||||
|
"producer_attempt_count": 1,
|
||||||
|
"terminal_action": "warn_continue"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -48,8 +62,11 @@ For another output module, its absence does not indicate a failed run.
|
|||||||
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
||||||
`output_directory` is the run-specific bundle, not the configured output root.
|
`output_directory` is the run-specific bundle, not the configured output root.
|
||||||
|
|
||||||
The receipt is a summary and discovery document. It does not contain lane
|
The receipt is a summary and discovery document. Its optional validation
|
||||||
descriptors, payloads, manifest data, rejections, warnings, or file contents.
|
summaries contain only stable status, identity, validator names, reason codes,
|
||||||
|
attempt counts, and terminal actions. It does not contain lane descriptors,
|
||||||
|
payloads, manifest payloads, rejection messages, warnings, raw model responses,
|
||||||
|
correction guidance, or file contents.
|
||||||
For the production JSON output, resolve `index_file` beneath
|
For the production JSON output, resolve `index_file` beneath
|
||||||
`output_directory`, reject path escapes, and use the
|
`output_directory`, reject path escapes, and use the
|
||||||
[Published JSON Output contract](json-output.md) to discover logical files and
|
[Published JSON Output contract](json-output.md) to discover logical files and
|
||||||
|
|||||||
@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
|
|||||||
|
|
||||||
## Dispatch And Configuration Handoff
|
## Dispatch And Configuration Handoff
|
||||||
|
|
||||||
The root dispatcher handles help, configuration validation, pipeline listing,
|
The root dispatcher handles help, version reporting, configuration validation,
|
||||||
and a pipeline run. It normalizes injectable options before dispatch so that a
|
pipeline listing, and a pipeline run. Version reporting resolves build
|
||||||
missing production dependency fails as a command error rather than reaching
|
information through `internal/buildinfo` before production composition, so the
|
||||||
execution.
|
diagnostic remains available without configuration or runtime collaborators.
|
||||||
|
The public syntax, streams, exit classes, and version semantics are defined by
|
||||||
|
the [CLI reference](../cli.md). Other root commands normalize injectable
|
||||||
|
options before dispatch so that a missing production dependency fails as a
|
||||||
|
command error rather than reaching execution.
|
||||||
|
|
||||||
Commands that need configuration use one shared loader. The CLI discovers the
|
Commands that need configuration use one shared loader. The CLI discovers the
|
||||||
file, parses it through **internal/core/config**, starts from defaults, applies
|
file, parses it through **internal/core/config**, starts from defaults, applies
|
||||||
|
|||||||
@@ -74,10 +74,19 @@ profile-free, and no second inheritance decision occurs during execution. The
|
|||||||
public field definitions and precedence are owned by
|
public field definitions and precedence are owned by
|
||||||
[Configuration](../config.md#pipelines).
|
[Configuration](../config.md#pipelines).
|
||||||
|
|
||||||
|
The resolver retains configured `validation_policy` overrides and derives one
|
||||||
|
detached concrete terminal policy for the chunk producer and every lane's
|
||||||
|
extract, merge, and normalize producers. That field-by-field inheritance is
|
||||||
|
complete before preparation, and the effective values contribute to pipeline
|
||||||
|
and checkpoint identity; execution does not interpret configuration defaults.
|
||||||
|
|
||||||
The framework resolver supplies defaults, selects lanes, resolves validator
|
The framework resolver supplies defaults, selects lanes, resolves validator
|
||||||
chains, checks registered module and artifact compatibility, validates module
|
chains, checks registered module and artifact compatibility, validates module
|
||||||
options, and returns the fixed ordered pipeline shape. The resulting
|
options, and returns the fixed ordered pipeline shape. Positive validator retry
|
||||||
**EffectiveConfig** retains the selected ID, requested selection and reference
|
budgets require an LLM-backed selected validator; deterministic validators are
|
||||||
|
rejected during resolution. Eligible LLM-backed producer specifications also
|
||||||
|
contribute their declared correction protocol to the resolved metadata. The
|
||||||
|
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
|
||||||
changes, a clone of the input configuration, and the resolved pipeline.
|
changes, a clone of the input configuration, and the resolved pipeline.
|
||||||
Callers may therefore retain or modify their input slices and maps without
|
Callers may therefore retain or modify their input slices and maps without
|
||||||
changing the resolved result, and later consumers cannot mutate the original
|
changing the resolved result, and later consumers cannot mutate the original
|
||||||
@@ -94,8 +103,8 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
|
|||||||
|
|
||||||
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
||||||
after defaults, lane selection, module bindings, reference bindings, validator
|
after defaults, lane selection, module bindings, reference bindings, validator
|
||||||
chains, effective LLM profiles, and artifact schema identity have been
|
chains, selected correction protocols, effective LLM profiles, and artifact
|
||||||
resolved. The digest excludes
|
schema identity have been resolved. The digest excludes
|
||||||
its own stored value. It identifies resolved composition rather than raw YAML
|
its own stored value. It identifies resolved composition rather than raw YAML
|
||||||
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
||||||
provenance before execution; cache and checkpoint identity have additional
|
provenance before execution; cache and checkpoint identity have additional
|
||||||
|
|||||||
@@ -33,7 +33,9 @@ typed builder. Scene chunking, every extractor, and NPC, location, and item-regi
|
|||||||
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
||||||
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
||||||
resolved-pipeline debug data and determines which selected bindings inherit the
|
resolved-pipeline debug data and determines which selected bindings inherit the
|
||||||
pipeline profile. Configuration remains the canonical owner of the exact keys,
|
pipeline profile. The registry normalizers use `single_response_v1`, forwarding
|
||||||
|
corrections to their reconciliation completion and retaining the accepted raw
|
||||||
|
proposal only as an owned model candidate. Configuration remains the canonical owner of the exact keys,
|
||||||
profile precedence, and validator order.
|
profile precedence, and validator order.
|
||||||
|
|
||||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||||
@@ -42,6 +44,13 @@ unknown fields, while preserving semantic candidates for deterministic
|
|||||||
validation. Do not promote a private response envelope into a durable schema;
|
validation. Do not promote a private response envelope into a durable schema;
|
||||||
the contracts above define durable data.
|
the contracts above define durable data.
|
||||||
|
|
||||||
|
A D&D producer that declares `single_response_v1` forwards any supplied
|
||||||
|
semantic correction to its structured completion and returns an owned copy of
|
||||||
|
that completion's exact validated raw response as its model candidate. It does
|
||||||
|
not serialize normalized artifacts to create that candidate, so deterministic
|
||||||
|
identity, evidence, warning, and durable-schema behavior remains separate from
|
||||||
|
the model transport material.
|
||||||
|
|
||||||
## Prompt Construction
|
## Prompt Construction
|
||||||
|
|
||||||
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
|
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
|
||||||
@@ -125,6 +134,13 @@ relatedness validators report advisory evidence concerns. The configured order
|
|||||||
is documented in
|
is documented in
|
||||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||||
|
|
||||||
|
Every D&D rejection describes the correction in transcript-grounded domain
|
||||||
|
terms, using contextual names, artifact fields, and source segment ranges when
|
||||||
|
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
||||||
|
hashes, validator module keys, or reason codes. Those identifiers remain in
|
||||||
|
ordinary validation provenance; only the actionable semantic guidance is
|
||||||
|
eligible for the correction prompt.
|
||||||
|
|
||||||
Enemy-event extraction additionally rejects a second `engaged` observation for
|
Enemy-event extraction additionally rejects a second `engaged` observation for
|
||||||
the same comparison identity within one scene-scoped result. Normalization may
|
the same comparison identity within one scene-scoped result. Normalization may
|
||||||
combine results from distinct scenes, so it intentionally does not apply that
|
combine results from distinct scenes, so it intentionally does not apply that
|
||||||
|
|||||||
@@ -42,6 +42,22 @@ observability. The adapter returns PromptKit’s validated raw bytes rather than
|
|||||||
re-encoding the decoded target. An empty optional material is represented as
|
re-encoding the decoded target. An empty optional material is represented as
|
||||||
one space so its named input is retained by PromptKit.
|
one space so its named input is retained by PromptKit.
|
||||||
|
|
||||||
|
When a request includes semantic correction material, the adapter validates and
|
||||||
|
defensively copies it before preparation, then appends exactly two messages
|
||||||
|
after the ordinarily rendered prompt: the prior response as an assistant
|
||||||
|
message and the correction guidance as a user message. Requests without a
|
||||||
|
correction do not add messages or introduce caller roles. Ordinary request
|
||||||
|
summaries record correction byte counts and digests only; complete messages are
|
||||||
|
available solely in an explicitly requested debug trace.
|
||||||
|
|
||||||
|
The adapter leaves the ordinary rendered message prefix, named inputs,
|
||||||
|
variables, session, profile, execution overrides, prepared-execution path, and
|
||||||
|
PromptKit repair policy unchanged for a corrected request. It never imports a
|
||||||
|
PromptKit message type into a module or pipeline contract. PromptKit reports
|
||||||
|
actual structural repair count and cumulative token usage per completion; the
|
||||||
|
pipeline's safe terminal debug record projects those values without copying
|
||||||
|
message content.
|
||||||
|
|
||||||
Client construction may also receive a run-wide reasoning-effort override from
|
Client construction may also receive a run-wide reasoning-effort override from
|
||||||
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
||||||
creates a fresh PromptKit execution override for each request: a nil pointer
|
creates a fresh PromptKit execution override for each request: a nil pointer
|
||||||
@@ -75,7 +91,7 @@ backend membership as runtime without performing generation. Fallback assets
|
|||||||
are mounted only when at least one source is registered. The production D&D
|
are mounted only when at least one source is registered. The production D&D
|
||||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||||
profile parsing: an operator-provided matching profile takes precedence over a
|
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||||
fallback profile without Notarius merging either document.
|
fallback profile without Notarius merging either document.
|
||||||
When the registration is absent, a profile selecting `backend: local` fails
|
When the registration is absent, a profile selecting `backend: local` fails
|
||||||
inspection instead of falling back to a built-in or endpoint-only target.
|
inspection instead of falling back to a built-in or endpoint-only target.
|
||||||
@@ -210,13 +226,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
|
|||||||
the pipeline's existing binding attempt policy sees the operational error and
|
the pipeline's existing binding attempt policy sees the operational error and
|
||||||
decides whether to rerun the complete operation.
|
decides whether to rerun the complete operation.
|
||||||
|
|
||||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
PromptKit executes structural repair within its structured-output flow. The
|
||||||
The current production D&D prompt manifests set repair attempts to zero. That
|
maintained production prompt manifests declare one additional repair attempt.
|
||||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
When a resolved binding supplies a repair value, the adapter inspects the
|
||||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
prompt, copies its complete output contract, changes only the repair limit, and
|
||||||
rejection is a recorded output rather than a provider error. The pipeline owns
|
passes that complete replacement contract to PromptKit. This preserves the
|
||||||
attempt lifecycle, validation chains, and retry diagnostics; see
|
prompt's output format, validation mode, schema, and provider structured-output
|
||||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
settings.
|
||||||
|
|
||||||
|
A successful repair is an ordinary successful completion, not a warning. The
|
||||||
|
adapter reports PromptKit's actual repair count and its cumulative usage
|
||||||
|
directly, without adding the initial and corrective counts again. Debug prompt
|
||||||
|
material records the configured complete contract; debug response material
|
||||||
|
records the repaired response and actual validation result. If the repair
|
||||||
|
budget is exhausted, the adapter retains the final raw bytes and debug material
|
||||||
|
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
|
||||||
|
or corrective call remain provider-neutral operational errors with the same
|
||||||
|
redaction boundary.
|
||||||
|
|
||||||
|
Structural repair does not replace pipeline retry behavior: a binding's
|
||||||
|
configured retry count reruns its complete stage attempt after an operational
|
||||||
|
or structural error, module-requested retry, or actionable semantic rejection.
|
||||||
|
The pipeline owns attempt lifecycle, validation chains, and retry diagnostics;
|
||||||
|
see [Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||||
[binding reference](../config.md#module-bindings-and-validators).
|
[binding reference](../config.md#module-bindings-and-validators).
|
||||||
|
|
||||||
## Timeout Ownership
|
## Timeout Ownership
|
||||||
@@ -247,6 +279,13 @@ surfaced; when the completion already failed, its call error remains the
|
|||||||
result. Debug-bundle location, retention, and handling are operational concerns
|
result. Debug-bundle location, retention, and handling are operational concerns
|
||||||
documented in [Operations](../operations.md#debug-bundles).
|
documented in [Operations](../operations.md#debug-bundles).
|
||||||
|
|
||||||
|
The attempt-terminal summary is a separate safe trace record: it contains
|
||||||
|
attempt kinds, validator outcome counts and reason codes, effective policy,
|
||||||
|
terminal action, and repair/usage references. It excludes raw assistant
|
||||||
|
responses and correction text. Those values can appear only in the explicitly
|
||||||
|
requested detailed prompt and response artifacts, which require sensitive-data
|
||||||
|
handling.
|
||||||
|
|
||||||
Run manifests receive selected profile summaries, including optional effective
|
Run manifests receive selected profile summaries, including optional effective
|
||||||
backend and reasoning provenance, and component identities—not prompt, schema,
|
backend and reasoning provenance, and component identities—not prompt, schema,
|
||||||
source, reference, or response content. The published field semantics belong
|
source, reference, or response content. The published field semantics belong
|
||||||
@@ -256,6 +295,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
|
|||||||
available to other runtime collaborators; it does not make prompt or response
|
available to other runtime collaborators; it does not make prompt or response
|
||||||
contents safe for general logging.
|
contents safe for general logging.
|
||||||
|
|
||||||
|
Generation failures expose an application-owned category and optional HTTP
|
||||||
|
status. Provider code, type, and message remain debug-only, after redaction.
|
||||||
|
|
||||||
## Failure Boundaries
|
## Failure Boundaries
|
||||||
|
|
||||||
- Construction fails for missing asset registries, mutually exclusive profile
|
- Construction fails for missing asset registries, mutually exclusive profile
|
||||||
|
|||||||
@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
|
|||||||
binding-specific profile on a deterministic module. The user-facing precedence
|
binding-specific profile on a deterministic module. The user-facing precedence
|
||||||
contract belongs in [Configuration](../config.md#pipelines).
|
contract belongs in [Configuration](../config.md#pipelines).
|
||||||
|
|
||||||
|
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
|
||||||
|
declare correction protocol `single_response_v1`. That declaration is a
|
||||||
|
promise that the implementation accepts one attempt-local semantic correction
|
||||||
|
and returns an owned copy of the exact one model response that directly
|
||||||
|
controlled the candidate. It must forward correction only to its structured
|
||||||
|
completion request; it must not manufacture prior-response material by
|
||||||
|
serializing a normalized artifact or expose opaque application IDs. Input,
|
||||||
|
output, validator, and deterministic specs cannot declare the protocol.
|
||||||
|
|
||||||
Implementations that accept options must provide both an option validator and
|
Implementations that accept options must provide both an option validator and
|
||||||
a builder. The validator is used while resolving configuration; the builder
|
a builder. The validator is used while resolving configuration; the builder
|
||||||
decodes the same options and constructs the implementation from the prepared
|
decodes the same options and constructs the implementation from the prepared
|
||||||
@@ -36,13 +45,23 @@ they need, register each leaf implementation, and add any family-owned assets
|
|||||||
or default validator chains. They return contextual errors so production
|
or default validator chains. They return contextual errors so production
|
||||||
composition fails at startup rather than at the first run.
|
composition fails at startup rather than at the first run.
|
||||||
|
|
||||||
|
A validator that returns a completed rejection must supply two separate
|
||||||
|
bounded values: a stable `ReasonCode` for provenance and actionable
|
||||||
|
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
|
||||||
|
and the constraints on one complete corrected replacement. It must not contain
|
||||||
|
validator keys, diagnostic paths, opaque application IDs, or other internal
|
||||||
|
identifiers. An operator-facing `Message` may explain the same event, but the
|
||||||
|
framework never copies it into a model request. Missing or invalid guidance is
|
||||||
|
a validator contract failure.
|
||||||
|
|
||||||
An artifact family can register an optional typed evidence projector alongside
|
An artifact family can register an optional typed evidence projector alongside
|
||||||
its codec. The projector returns defensive copies of the artifact's direct
|
its codec. The projector returns defensive copies of the artifact's direct
|
||||||
generic source references and must use the codec's exact Go type. It does not
|
generic source references and must use the codec's exact Go type. It does not
|
||||||
interpret surrounding context or publish files; the pipeline validates the
|
interpret surrounding context or publish files; the pipeline validates the
|
||||||
capability during preparation and the output boundary owns publication. See
|
capability during preparation and the output boundary owns publication. See
|
||||||
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
||||||
for the durable result.
|
for the durable source-unit excerpt. Lane artifacts retain citation and lane
|
||||||
|
provenance; the framework does not add either to that published excerpt.
|
||||||
|
|
||||||
An artifact family is broader than a module: it owns the cohesive domain
|
An artifact family is broader than a module: it owns the cohesive domain
|
||||||
feature across its artifact type, codec, stage modules, validators, prompt
|
feature across its artifact type, codec, stage modules, validators, prompt
|
||||||
@@ -90,6 +109,12 @@ combined-material bound preserves the deterministic result under the family's
|
|||||||
fallback policy. Provider, transport, cancellation, and context-construction
|
fallback policy. Provider, transport, cancellation, and context-construction
|
||||||
failures remain execution errors.
|
failures remain execution errors.
|
||||||
|
|
||||||
|
When the engine actually makes a proposal call, its typed result carries the
|
||||||
|
owned exact proposal response under the same correction contract as other
|
||||||
|
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
|
||||||
|
model candidate, so a later rejection applies terminal policy without spending
|
||||||
|
an ineffective semantic retry.
|
||||||
|
|
||||||
The core supplies a conservative generic prompt and the single private
|
The core supplies a conservative generic prompt and the single private
|
||||||
response schema. A domain prompt may substitute its semantic instructions but
|
response schema. A domain prompt may substitute its semantic instructions but
|
||||||
mounts the core-owned protocol and candidate/transcript presentation assets.
|
mounts the core-owned protocol and candidate/transcript presentation assets.
|
||||||
@@ -110,9 +135,13 @@ its domain prompt.
|
|||||||
3. Implement strict option decoding, construction, and the typed stage
|
3. Implement strict option decoding, construction, and the typed stage
|
||||||
interface. Preserve caller ownership: do not retain mutable request data
|
interface. Preserve caller ownership: do not retain mutable request data
|
||||||
and return defensive copies where an implementation exposes stored data.
|
and return defensive copies where an implementation exposes stored data.
|
||||||
|
If declaring correction capability, forward the request correction and
|
||||||
|
retain only the exact validated response that controlled the result.
|
||||||
4. Register the module through its typed registry helper and add it to the
|
4. Register the module through its typed registry helper and add it to the
|
||||||
owning family registrar. Add a default validator chain only when that
|
owning family registrar. Add a default validator chain only when that
|
||||||
family owns the behavior; otherwise require an explicit compatible chain.
|
family owns the behavior; otherwise require an explicit compatible chain.
|
||||||
|
Every rejection path in a validator must provide actionable correction
|
||||||
|
guidance while retaining its stable internal reason code.
|
||||||
5. Update the selectable-key and chain reference in
|
5. Update the selectable-key and chain reference in
|
||||||
[Configuration](../config.md#production-module-keys), the applicable
|
[Configuration](../config.md#production-module-keys), the applicable
|
||||||
integration contract, and focused tests. Keep the configuration document
|
integration contract, and focused tests. Keep the configuration document
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ physical state roots.
|
|||||||
| Area | Implemented owners | Responsibility |
|
| Area | Implemented owners | Responsibility |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
||||||
|
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
|
||||||
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
||||||
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
||||||
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
||||||
|
|||||||
@@ -32,13 +32,26 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
|
|||||||
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
||||||
requested lanes where that is supported, resolves validator chains, checks
|
requested lanes where that is supported, resolves validator chains, checks
|
||||||
module capabilities and typed artifact compatibility, validates options, and
|
module capabilities and typed artifact compatibility, validates options, and
|
||||||
assigns a deterministic resolved-composition digest. The resolved pipeline
|
assigns a deterministic resolved-composition digest. A correction protocol is
|
||||||
contains bindings and declared reference targets, not external reference bytes.
|
selected from each eligible LLM-backed producer specification and becomes part
|
||||||
|
of that resolved identity; only `single_response_v1` is currently supported.
|
||||||
|
Preparation rejects an LLM-backed producer that combines a non-empty validator
|
||||||
|
chain with positive producer retries unless it declares that protocol. Producers
|
||||||
|
without validators or without retries remain valid without correction support.
|
||||||
|
The resolved pipeline contains bindings and declared reference targets, not
|
||||||
|
external reference bytes.
|
||||||
After selection, the resolver applies command, binding, and pipeline profile
|
After selection, the resolver applies command, binding, and pipeline profile
|
||||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||||
an empty resolved binding profile. Deterministic bindings remain profile-free.
|
an empty resolved binding profile. It resolves structural output repair
|
||||||
These effective values are part of the digest, so execution and checkpoint
|
separately: a binding's `structured_output_repair_attempts` value wins, then a
|
||||||
consumers do not repeat profile inheritance.
|
pipeline value applies to LLM-backed bindings and validators, and omission
|
||||||
|
leaves the prompt-owned policy intact. An explicit repair value on a
|
||||||
|
deterministic binding is rejected. Resolved bindings own copied repair values,
|
||||||
|
and these effective values are part of the digest, so execution and checkpoint
|
||||||
|
consumers do not repeat profile inheritance or configuration resolution.
|
||||||
|
Each LLM request receives its own copy of that resolved value. PromptKit spends
|
||||||
|
it only for structural correction inside one completion; the runner's binding
|
||||||
|
retry policy remains the separate outer budget for complete stage attempts.
|
||||||
Configuration resolution supplies the selected profile and catalog; see
|
Configuration resolution supplies the selected profile and catalog; see
|
||||||
[Configuration Internals](configuration.md).
|
[Configuration Internals](configuration.md).
|
||||||
|
|
||||||
@@ -52,9 +65,11 @@ remains declared but has no bytes until its producing step completes.
|
|||||||
|
|
||||||
Preparation is the construction boundary. It validates the resolved shape and
|
Preparation is the construction boundary. It validates the resolved shape and
|
||||||
registry set, clones the resolved data, then constructs the input adapter,
|
registry set, clones the resolved data, then constructs the input adapter,
|
||||||
chunker, stage-local validators, every typed lane, and output encoder. Each
|
chunker, stage-local validators, every typed lane, and output encoder. The
|
||||||
registered builder receives its own cloned build request immediately before its
|
prepared producer metadata preserves each selected correction protocol, and
|
||||||
module-owned code runs. Preparation also collects stable checkpoint
|
the resolved digest carrying that metadata participates in checkpoint identity.
|
||||||
|
Each registered builder receives its own cloned build request immediately
|
||||||
|
before its module-owned code runs. Preparation also collects stable checkpoint
|
||||||
fingerprints. Missing registrations, incompatible typed entries, nil
|
fingerprints. Missing registrations, incompatible typed entries, nil
|
||||||
implementations, and constructor failures are reported before source parsing
|
implementations, and constructor failures are reported before source parsing
|
||||||
or any stage operation begins.
|
or any stage operation begins.
|
||||||
@@ -121,14 +136,45 @@ their target: chunks, codec-decoded typed candidates, or serialized codec
|
|||||||
bytes. Each typed validator receives a newly decoded value from the one
|
bytes. Each typed validator receives a newly decoded value from the one
|
||||||
candidate serialization for that attempt, while serialized validators receive
|
candidate serialization for that attempt, while serialized validators receive
|
||||||
separately owned representation bytes and schema metadata. They may approve,
|
separately owned representation bytes and schema metadata. They may approve,
|
||||||
approve with warnings, reject, or fail. A rejection is an ordinary pipeline
|
approve with warnings, reject, fail, or be skipped when a runtime prerequisite
|
||||||
result; a validator error is a framework error.
|
is unavailable. The shared executor settles every configured validator in
|
||||||
|
order. A failed LLM-backed validator retries only itself against the same
|
||||||
|
immutable candidate; it does not regenerate the producer or alter the
|
||||||
|
validator request. Rejections stop that validator, while other configured
|
||||||
|
validators still run. The executor retains ordered results, bounded
|
||||||
|
deduplicated correction guidance from rejections, and only the final exhausted
|
||||||
|
failure outcome for each validator. The correction builder keeps first
|
||||||
|
occurrence order, omits internal reason codes, validator names, and operator
|
||||||
|
messages, and requests one complete replacement. Missing guidance or an
|
||||||
|
oversized aggregate is a framework contract error; guidance is never inferred
|
||||||
|
or truncated.
|
||||||
|
|
||||||
The runner applies the binding's retry policy around a stage operation and its
|
The runner applies the binding's retry policy around a stage operation and its
|
||||||
complete validation chain. It preserves warnings only from the final accepted
|
complete validation chain. It preserves warnings only from the final accepted
|
||||||
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
|
or rejected attempt, plus one fixed warning per validator whose execution
|
||||||
directives consume this same budget and validate any final safe fallback through
|
budget was exhausted under `warn_continue`. Cancellation stops retries.
|
||||||
the normalizer chain.
|
Normalizer-specific retry directives consume this same budget and validate any
|
||||||
|
final safe fallback through the normalizer chain.
|
||||||
|
|
||||||
|
The artifact-neutral producer-attempt state machine owns that shared budget,
|
||||||
|
attempt provenance, semantic-correction material, and terminal-policy
|
||||||
|
selection. It accepts producer and complete-validation closures, so artifact
|
||||||
|
materialization, cache handling, checkpoints, and debug output stay at the
|
||||||
|
operation boundary. It distinguishes operational, structural, module-requested,
|
||||||
|
and semantic retries. A semantic retry is available only for a valid latest
|
||||||
|
`single_response_v1` candidate; a deterministic or no-model rejection instead
|
||||||
|
settles the semantic policy immediately. Structural-output errors alone use the
|
||||||
|
structural policy, and validation failure without rejection settles the
|
||||||
|
validator-failure policy without regenerating the producer.
|
||||||
|
|
||||||
|
Chunk planning uses this state machine for generated plans. A rejected or
|
||||||
|
validation-incomplete automatic cache hit is not model material and therefore
|
||||||
|
falls through to a fresh initial generation at producer attempt one; it neither
|
||||||
|
receives a correction, consumes retry budget, promotes cached-candidate
|
||||||
|
warnings, nor overwrites the stored record. An incomplete cache validation
|
||||||
|
under `fail_run` terminates instead. Only a newly generated, completely
|
||||||
|
validated plan is published to the chunk-plan store. Rejected plans never
|
||||||
|
advance, and validation-incomplete plans remain unpublishable.
|
||||||
|
|
||||||
After terminal lane work, the runner assembles manifest provenance, normalized
|
After terminal lane work, the runner assembles manifest provenance, normalized
|
||||||
artifacts, rejections, warnings, and an optional accepted chunk map. When an
|
artifacts, rejections, warnings, and an optional accepted chunk map. When an
|
||||||
@@ -142,6 +188,15 @@ The CLI publishes those files only after the runner returns without a framework
|
|||||||
error. Logical file names and schemas are defined by the [output integration
|
error. Logical file names and schemas are defined by the [output integration
|
||||||
contracts](../integrations/).
|
contracts](../integrations/).
|
||||||
|
|
||||||
|
For every completed producer disposition, the runner projects one bounded
|
||||||
|
validation summary to the manifest, the affected rejection when present, and
|
||||||
|
the CLI result receipt. The summary records status, configured-order rejecting
|
||||||
|
validators and reason codes, incomplete validators, producer-attempt count,
|
||||||
|
and terminal action. It contains no operator message, correction guidance, or
|
||||||
|
model response. `complete`, `rejected`, and `incomplete` describe the final
|
||||||
|
candidate disposition; a run-level `incomplete` status indicates at least one
|
||||||
|
current-run output advanced under `warn_continue`.
|
||||||
|
|
||||||
## Checkpoint And Debug Hooks
|
## Checkpoint And Debug Hooks
|
||||||
|
|
||||||
The runner receives checkpoint and debug interfaces rather than roots. It
|
The runner receives checkpoint and debug interfaces rather than roots. It
|
||||||
@@ -151,6 +206,20 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
|
|||||||
Selective recomputation can require a canonical accepted normalized predecessor
|
Selective recomputation can require a canonical accepted normalized predecessor
|
||||||
before a dependent lane starts.
|
before a dependent lane starts.
|
||||||
|
|
||||||
|
The runner writes successful checkpoint artifacts only after complete accepted
|
||||||
|
validation. Chunk plans follow the same rule for publication. A rejection,
|
||||||
|
invalid structured response, or incomplete validation is never reusable state;
|
||||||
|
the current run may still hand off an otherwise valid `warn_continue` result
|
||||||
|
according to its terminal policy. The runner carries private reuse eligibility
|
||||||
|
through extract, merge, normalize, and generated-reference handoff. Any stage
|
||||||
|
derived from incomplete validation skips both checkpoint lookup and all
|
||||||
|
checkpoint publication even when that stage's own validation completes.
|
||||||
|
External references and fully validated generated references remain eligible.
|
||||||
|
Attempt debug records retain safe kind,
|
||||||
|
validator, repair-usage, policy, and terminal-decision provenance. Full
|
||||||
|
assistant and correction content remains confined to the requested detailed
|
||||||
|
LLM trace.
|
||||||
|
|
||||||
Debug recording is attempt-scoped and application-owned. A failure to persist
|
Debug recording is attempt-scoped and application-owned. A failure to persist
|
||||||
required debug data is a framework error. State roots, persistence, reason-code
|
required debug data is a framework error. State roots, persistence, reason-code
|
||||||
meanings, resume, and cleanup are intentionally owned by
|
meanings, resume, and cleanup are intentionally owned by
|
||||||
|
|||||||
@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
|
|||||||
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
||||||
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
||||||
|
|
||||||
|
## Source Deployment
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. Install a pinned source release
|
||||||
|
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Pin the exact tag in deployment automation rather than following a branch.
|
||||||
|
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
|
||||||
|
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
|
||||||
|
The maintainer publication process, including tag guards and verification,
|
||||||
|
belongs to [Source Releases](release.md). macOS builds are best-effort for
|
||||||
|
development, and Windows is unsupported.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
Each run can use independent roots with different retention and access-control
|
Each run can use independent roots with different retention and access-control
|
||||||
@@ -72,6 +89,9 @@ provider call or credentials:
|
|||||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
|
An unset optional `api_key_env` reaches the provider without authorization and
|
||||||
|
may receive a 401 or 403 response.
|
||||||
|
|
||||||
Profile paths are currently resolved from the process working directory, not
|
Profile paths are currently resolved from the process working directory, not
|
||||||
from the configuration file. The complete example's
|
from the configuration file. The complete example's
|
||||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||||
@@ -89,6 +109,33 @@ On success, the command reports the output bundle path. A warning-bearing run
|
|||||||
still succeeds and reports its warning count on standard error. Errors and
|
still succeeds and reports its warning count on standard error. Errors and
|
||||||
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
||||||
|
|
||||||
|
## Validation Retries And Terminal Outcomes
|
||||||
|
|
||||||
|
Each producer binding has one outer **retries** budget. It covers complete
|
||||||
|
producer attempts for operational failures, invalid structured output,
|
||||||
|
normalizer fallback retry, and semantic correction. It is independent from
|
||||||
|
PromptKit's structural-repair calls inside one completion and from an
|
||||||
|
LLM-backed validator's own retry budget. A semantic correction rebuilds the
|
||||||
|
ordinary producer request and supplies only the latest rejected model response
|
||||||
|
plus aggregated validator guidance; it is not a conversation replay.
|
||||||
|
|
||||||
|
After the applicable budgets are exhausted, the resolved
|
||||||
|
[`validation_policy`](config.md#pipelines) determines the result. Structural
|
||||||
|
failure and semantic rejection normally fail the run; an explicit
|
||||||
|
`reject_output` records a rejection and allows unrelated work to finish. A
|
||||||
|
validator execution failure normally uses `warn_continue`, which keeps an
|
||||||
|
otherwise accepted result in the current run with `incomplete` validation
|
||||||
|
provenance. It emits one bounded warning for every validator whose execution
|
||||||
|
budget was exhausted. A corrected result that later passes validation does not
|
||||||
|
retain abandoned-attempt warnings.
|
||||||
|
|
||||||
|
Treat a successful process exit as a completed run, not as proof that every
|
||||||
|
candidate was fully validated. Inspect the receipt's `validation_status`,
|
||||||
|
`validation_summaries`, rejection count, and warning count when an orchestrator
|
||||||
|
requires complete validation. The durable fields and their meanings are owned
|
||||||
|
by the [run-result receipt](integrations/run-result.md) and
|
||||||
|
[published JSON output contract](integrations/json-output.md).
|
||||||
|
|
||||||
## Output Bundles
|
## Output Bundles
|
||||||
|
|
||||||
Each successful run receives a generated safe run identifier and writes beneath:
|
Each successful run receives a generated safe run identifier and writes beneath:
|
||||||
@@ -110,9 +157,9 @@ are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
|
|||||||
[evidence context](integrations/evidence-context.md) contains source-unit text
|
[evidence context](integrations/evidence-context.md) contains source-unit text
|
||||||
and metadata. It is not a cache or debug artifact: retain it with the output
|
and metadata. It is not a cache or debug artifact: retain it with the output
|
||||||
bundle only for as long as consumers need it, and apply source-content access
|
bundle only for as long as consumers need it, and apply source-content access
|
||||||
controls to the entire bundle. Selected lanes may collectively cite most of a
|
controls to the entire bundle. Its selected source-unit excerpt may include
|
||||||
transcript, so a broad allowlist can make the evidence artifact nearly as
|
every source unit once when coverage is broad or its configured window is
|
||||||
sensitive and large as the source itself.
|
large, so do not assume a byte or token reduction or reduced sensitivity.
|
||||||
|
|
||||||
## Chunk-Plan Cache
|
## Chunk-Plan Cache
|
||||||
|
|
||||||
@@ -138,7 +185,9 @@ The configured cache mode controls one invocation:
|
|||||||
A reused plan is still materialized and validated against the current source.
|
A reused plan is still materialized and validated against the current source.
|
||||||
If a prior plan no longer gives acceptable results, use a refresh run rather
|
If a prior plan no longer gives acceptable results, use a refresh run rather
|
||||||
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
||||||
chunking work.
|
chunking work. A plan accepted only under incomplete validation is not
|
||||||
|
published, and a rejected cache hit falls through to ordinary generation rather
|
||||||
|
than becoming a correction candidate.
|
||||||
|
|
||||||
## Checkpoint Recording, Resume, And Recompute
|
## Checkpoint Recording, Resume, And Recompute
|
||||||
|
|
||||||
@@ -163,6 +212,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
|
|||||||
runtime identities, so checkpoints created under one state are not reused by
|
runtime identities, so checkpoints created under one state are not reused by
|
||||||
either of the others.
|
either of the others.
|
||||||
|
|
||||||
|
Only accepted, completely validated chunk, extract, merge, and normalize
|
||||||
|
results are checkpointed for reuse. Rejected, structurally invalid, and
|
||||||
|
validation-incomplete producer results remain non-reusable, even when a
|
||||||
|
`warn_continue` result advanced during its original run. A resumed invocation
|
||||||
|
therefore reruns that producer rather than treating degraded state as accepted.
|
||||||
|
|
||||||
Checkpoint state is confined below an identity-specific path:
|
Checkpoint state is confined below an identity-specific path:
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
@@ -217,13 +272,16 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
|
|||||||
~~~
|
~~~
|
||||||
|
|
||||||
The summary contains redacted invocation and resolution information plus run,
|
The summary contains redacted invocation and resolution information plus run,
|
||||||
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
|
warning, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
|
||||||
contains allowlisted application diagnostic records and can include source or
|
terminal records contain bounded attempt kinds, validator outcomes, policy,
|
||||||
derived application data. Neither surface is a cache input. Do not treat a
|
decision, PromptKit repair count, and usage; they do not contain assistant
|
||||||
debug bundle as safe to share merely because its configuration summary is
|
responses or complete correction messages. The trace contains allowlisted
|
||||||
redacted. Invocation metadata omits reasoning effort when it is inherited,
|
application diagnostic records and can include source, model, and correction
|
||||||
records the replacement value when one is supplied, and records an empty value
|
content. Neither surface is a cache input. Do not treat a debug bundle as safe
|
||||||
when inherited reasoning was explicitly cleared.
|
to share merely because its configuration summary is redacted. Invocation
|
||||||
|
metadata omits reasoning effort when it is inherited, records the replacement
|
||||||
|
value when one is supplied, and records an empty value when inherited reasoning
|
||||||
|
was explicitly cleared.
|
||||||
|
|
||||||
Notarius never creates debug state without an explicit request and never
|
Notarius never creates debug state without an explicit request and never
|
||||||
automatically deletes a requested bundle. If allocation succeeds, the command
|
automatically deletes a requested bundle. If allocation succeeds, the command
|
||||||
@@ -255,14 +313,29 @@ Provider execution settings and the generation timeout come from the selected
|
|||||||
PromptKit profile. The invocation-only **--reasoning-effort** and
|
PromptKit profile. The invocation-only **--reasoning-effort** and
|
||||||
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
||||||
for all LLM-backed calls in one run without changing the profile. PromptKit
|
for all LLM-backed calls in one run without changing the profile. PromptKit
|
||||||
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
|
structural output repair happens within one structured-completion call. Its
|
||||||
complete module operation and validation chain as defined by
|
effective `structured_output_repair_attempts` limit is resolved from the
|
||||||
[module bindings](config.md#module-bindings-and-validators).
|
selected binding, then the pipeline, then the prompt declaration; see
|
||||||
|
[module bindings](config.md#module-bindings-and-validators). This is distinct
|
||||||
|
from Notarius binding **retries**, which rerun the complete module operation
|
||||||
|
and validation chain and do not consume or replenish the structural-repair
|
||||||
|
limit. The maintained production prompts declare one repair attempt, paid only
|
||||||
|
after a structural failure. One structured completion with repair budget **R**
|
||||||
|
makes at most **R + 1** serial provider calls. If one stage attempt performs
|
||||||
|
**C** structured completions, a binding with **retries: N** has a maximum of
|
||||||
|
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
|
||||||
|
corresponding invocation counts and budgets. This is an upper bound, not a
|
||||||
|
promise that every call reaches a provider.
|
||||||
|
|
||||||
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
||||||
effective generation timeout adds an inner request deadline, while zero
|
effective generation timeout adds an inner request deadline, while zero
|
||||||
disables only that generation deadline. The HTTP client timeout remains a
|
disables only that generation deadline. The HTTP client timeout remains a
|
||||||
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
||||||
|
Repairs are serial within the same caller context, so their worst-case latency
|
||||||
|
and cost follow the provider-call bound above; provision run deadlines and
|
||||||
|
provider budgets accordingly. Credentials remain optional unless the selected
|
||||||
|
PromptKit profile requires one, in which case preparation fails before a
|
||||||
|
provider call when its configured credential is unavailable.
|
||||||
The pinned upstream boundary and profile-format links are in
|
The pinned upstream boundary and profile-format links are in
|
||||||
[PromptKit Integration](integrations/pkg-promptkit.md).
|
[PromptKit Integration](integrations/pkg-promptkit.md).
|
||||||
|
|
||||||
@@ -279,6 +352,11 @@ positive value makes the effective active local-generation bound the smaller
|
|||||||
of **total_llm** and that local limit, so a local limit of four permits no more
|
of **total_llm** and that local limit, so a local limit of four permits no more
|
||||||
than four active local generations.
|
than four active local generations.
|
||||||
|
|
||||||
|
The Notarius scheduler admits one logical structured completion and holds that
|
||||||
|
permit while PromptKit performs its serial corrective calls. PromptKit applies
|
||||||
|
its selected-backend admission to each provider call; Notarius does not
|
||||||
|
reacquire a permit or add another scheduler for a repair.
|
||||||
|
|
||||||
For a positive local limit, PromptKit owns its default waiting capacity and
|
For a positive local limit, PromptKit owns its default waiting capacity and
|
||||||
admission behavior. When a PromptKit backend has admitted all active and queued
|
admission behavior. When a PromptKit backend has admitted all active and queued
|
||||||
work, a new call fails as capacity exhaustion before generation. The adapter
|
work, a new call fails as capacity exhaustion before generation. The adapter
|
||||||
|
|||||||
@@ -167,18 +167,29 @@ starting, waits for started work, and prevents output encoding.
|
|||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
Validation is a framework-managed boundary around outputs from chunk, extract,
|
Validation is a framework-managed boundary around outputs from chunk, extract,
|
||||||
merge, and normalize stages. Validators receive immutable stage output
|
merge, and normalize stages. Validators receive immutable stage output and
|
||||||
and make an explicit whole-output decision: approve, approve with warnings, or
|
make an explicit whole-output decision: approve, approve with warnings,
|
||||||
reject.
|
reject, fail, or skip when a runtime prerequisite is unavailable.
|
||||||
|
|
||||||
Typed artifact validators receive the domain value directly. Chunk validators
|
Typed artifact validators receive the domain value directly. Chunk validators
|
||||||
receive source-zone chunks, while serialized validators receive immutable
|
receive source-zone chunks, while serialized validators receive immutable
|
||||||
representation bytes and declared schema metadata. A validator registered for
|
representation bytes and declared schema metadata. A validator registered for
|
||||||
one target or artifact kind cannot satisfy an incompatible selection.
|
one target or artifact kind cannot satisfy an incompatible selection.
|
||||||
|
|
||||||
Rejection is a recorded pipeline outcome, not a framework execution error.
|
The framework runs every applicable validator sequentially in configured order.
|
||||||
Validator execution failures are framework errors. Rejected output does not
|
It aggregates rejections, exhausted validator failures, and skips before the
|
||||||
advance to the next stage.
|
producer policy chooses a disposition. A completed rejection never advances.
|
||||||
|
With no rejection, an exhausted validator failure may fail the run or, under
|
||||||
|
the configured `warn_continue` policy, advance a structurally valid candidate
|
||||||
|
with explicit incomplete-validation provenance. Validators report findings;
|
||||||
|
they do not choose candidate disposition.
|
||||||
|
|
||||||
|
A completed rejection supplies a stable reason code for internal provenance
|
||||||
|
and bounded actionable correction guidance for the candidate producer. Reason
|
||||||
|
codes, validator keys, and operator-facing messages remain diagnostic data;
|
||||||
|
they are not model instructions. The framework constructs model-facing retry
|
||||||
|
text only from the semantic guidance and fails the contract rather than
|
||||||
|
inventing or truncating missing guidance.
|
||||||
|
|
||||||
Default validator chains are production composition policy and are registered
|
Default validator chains are production composition policy and are registered
|
||||||
centrally by stage and module. Configuration may replace a stage-local default,
|
centrally by stage and module. Configuration may replace a stage-local default,
|
||||||
@@ -195,6 +206,19 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
|
|||||||
and interpretation of structured output. Provider adapters do not own source-
|
and interpretation of structured output. Provider adapters do not own source-
|
||||||
or domain-specific prompt logic.
|
or domain-specific prompt logic.
|
||||||
|
|
||||||
|
PromptKit owns bounded structural correction within one structured completion.
|
||||||
|
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
|
||||||
|
the two budgets must remain separate.
|
||||||
|
|
||||||
|
An LLM-backed producer can participate in semantic correction only when it
|
||||||
|
declares `single_response_v1` and returns the exact one response that directly
|
||||||
|
controlled its candidate. On an actionable rejection, the framework rebuilds
|
||||||
|
the ordinary request and appends only the latest defective response as an
|
||||||
|
`assistant` message plus one aggregated `user` correction message. This is a
|
||||||
|
fresh replacement request, not a growing conversation. The retry budgets,
|
||||||
|
terminal policy, and sensitive-data rationale are recorded in
|
||||||
|
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md).
|
||||||
|
|
||||||
When a model selects an application entity, callers must supply a contextual
|
When a model selects an application entity, callers must supply a contextual
|
||||||
selection and deterministically attach the opaque application identity whenever
|
selection and deterministically attach the opaque application identity whenever
|
||||||
the selection resolves exactly. Models do not receive or reproduce opaque
|
the selection resolves exactly. Models do not receive or reproduce opaque
|
||||||
@@ -229,7 +253,8 @@ invalid or incompatible.
|
|||||||
|
|
||||||
Run manifests record enough resolved pipeline, module, source, reference, and
|
Run manifests record enough resolved pipeline, module, source, reference, and
|
||||||
LLM provenance to make a run auditable after configuration changes. Manifests
|
LLM provenance to make a run auditable after configuration changes. Manifests
|
||||||
record identities and summaries rather than secret or large payload content.
|
record identities and bounded validation summaries rather than secret, raw
|
||||||
|
model, correction, or large payload content.
|
||||||
|
|
||||||
## State, Output, And Safety
|
## State, Output, And Safety
|
||||||
|
|
||||||
@@ -249,18 +274,39 @@ an invocation that explicitly requests resume. Debug is never a cache input and
|
|||||||
is never created without an explicit request. Pipeline modules receive
|
is never created without an explicit request. Pipeline modules receive
|
||||||
collaborator interfaces and never physical roots.
|
collaborator interfaces and never physical roots.
|
||||||
|
|
||||||
|
Only accepted, completely validated producer output is reusable checkpoint or
|
||||||
|
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
|
||||||
|
results cannot become cache or checkpoint inputs, even when a
|
||||||
|
`warn_continue` result is allowed to advance in the current run. This
|
||||||
|
ineligibility follows derived merge and normalize results and generated
|
||||||
|
references for the remainder of the run: current-run handoff remains allowed,
|
||||||
|
but no dependent cache or checkpoint may be loaded or published.
|
||||||
|
|
||||||
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
||||||
deletion must be narrow and explicit. Notarius never automatically deletes
|
deletion must be narrow and explicit. Notarius never automatically deletes
|
||||||
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
||||||
|
|
||||||
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
||||||
traces, manifests, documentation, examples, or redacted configuration. Debug
|
traces, manifests, documentation, examples, or redacted configuration. Raw
|
||||||
|
assistant responses and complete correction messages are attempt-local and are
|
||||||
|
excluded from ordinary durable records and summaries; the requested detailed
|
||||||
|
debug trace is the sole diagnostic surface allowed to retain them. Debug
|
||||||
collection is allowlisted to application-owned payloads and must not capture
|
collection is allowlisted to application-owned payloads and must not capture
|
||||||
unrelated process environment values or filesystem content. Trace data may
|
unrelated process environment values or filesystem content. Trace data may
|
||||||
contain application data and therefore inherits its sensitivity; operators own
|
contain application data and therefore inherits its sensitivity; operators own
|
||||||
access controls and retention. Physical layout and operation are defined in
|
access controls and retention. Physical layout and operation are defined in
|
||||||
[Operations](../operations.md).
|
[Operations](../operations.md).
|
||||||
|
|
||||||
|
## Platform And Distribution
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. macOS is supported only as a
|
||||||
|
best-effort development and compilation environment, while Windows is
|
||||||
|
unsupported. Notarius distributes source releases only: an immutable source
|
||||||
|
tag and its checked-in release note identify a release. The project does not
|
||||||
|
publish executable binaries, archives, installers, container images,
|
||||||
|
checksums, signatures, or package-manager entries. Maintainer release commands
|
||||||
|
and tag guards belong to [Source Releases](../release.md).
|
||||||
|
|
||||||
## Architectural Non-Goals
|
## Architectural Non-Goals
|
||||||
|
|
||||||
Notarius does not aim to provide:
|
Notarius does not aim to provide:
|
||||||
|
|||||||
@@ -65,6 +65,8 @@ secret values.
|
|||||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
|
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
|
||||||
|
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
|
|||||||
crosses these topics, choose the document that owns the task and link to the
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
other contracts.
|
other contracts.
|
||||||
|
|
||||||
|
### Releases
|
||||||
|
|
||||||
|
`docs/release.md` owns the source-release procedure. Release notes are
|
||||||
|
historical summaries, not current-state contract owners: the checked-in note at
|
||||||
|
an immutable tag records that release, while current canonical documentation
|
||||||
|
must change with the behavior it describes. Do not use a release note to defer
|
||||||
|
or replace current documentation updates.
|
||||||
|
|
||||||
### Contracts And Implementation
|
### Contracts And Implementation
|
||||||
|
|
||||||
Integration and API documents define externally observable shapes and
|
Integration and API documents define externally observable shapes and
|
||||||
|
|||||||
164
docs/release.md
Normal file
164
docs/release.md
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
# Source Releases
|
||||||
|
|
||||||
|
This procedure is for maintainers publishing Notarius source releases. A
|
||||||
|
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
|
||||||
|
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
|
||||||
|
candidate after publication; it does not publish or repair a release.
|
||||||
|
|
||||||
|
Notarius publishes no binaries, archives, checksums, signatures, containers,
|
||||||
|
package-manager entries, or Gitea release objects. Windows is not supported.
|
||||||
|
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
|
||||||
|
`v0.3.0` tags.
|
||||||
|
|
||||||
|
## Select And Describe The Release
|
||||||
|
|
||||||
|
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
|
||||||
|
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
|
||||||
|
documented CLI, configuration, durable artifact, integration, or operating
|
||||||
|
contract when its note explains the impact and required operator action. A
|
||||||
|
patch release must not intentionally break those documented contracts within
|
||||||
|
its minor line.
|
||||||
|
|
||||||
|
Create the version-matched note as part of the candidate. Every new note uses
|
||||||
|
this structure, with concise, truthful content in each section:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# Notarius vMAJOR.MINOR.PATCH
|
||||||
|
|
||||||
|
This release ...
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
```
|
||||||
|
|
||||||
|
The note is a historical summary. Link to current canonical documentation for
|
||||||
|
exact behavior, and update that documentation in the candidate rather than
|
||||||
|
using the note as a substitute.
|
||||||
|
|
||||||
|
## Prepare The Candidate
|
||||||
|
|
||||||
|
Set the selected release version and disable Go workspace use for every
|
||||||
|
candidate command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_VERSION=vMAJOR.MINOR.PATCH
|
||||||
|
export RELEASE_VERSION GOWORK=off
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the shared source-candidate checks from the repository. They cover module
|
||||||
|
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
|
||||||
|
configuration validation, and the Linux and Darwin command-build matrix:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./scripts/check-release-source.sh "$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Before committing, manually follow every changed local Markdown link and
|
||||||
|
review the candidate for unintended files, generated output, credentials, or
|
||||||
|
other unrelated changes. Commit the release note and all affected current
|
||||||
|
documentation, then run the shared checker against that exact candidate. Push
|
||||||
|
the candidate commit to `main` only after it succeeds. Record the exact commit
|
||||||
|
only after that push:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
For private-module installation, configure standard `GOPRIVATE` matching this
|
||||||
|
module and ordinary Git authentication for the hosting service before running
|
||||||
|
the verification below. The exact authentication mechanism belongs to the
|
||||||
|
maintainer environment; never record credentials or environment dumps in a
|
||||||
|
release note, command history, or repository file.
|
||||||
|
|
||||||
|
## Guard And Publish The Tag
|
||||||
|
|
||||||
|
Fetch current remote references, then run this guard without editing the
|
||||||
|
candidate. It requires `main`, a clean worktree and index, disabled workspace
|
||||||
|
use, a stable release version, the recorded and pushed commit, a matching note,
|
||||||
|
and unused local and remote tags:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git fetch origin main --tags
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
test "$GOWORK" = off
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
|
||||||
|
test -s "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
done
|
||||||
|
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Create an explicitly lightweight tag against the guarded commit, verify its
|
||||||
|
target, and push only that tag ref:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
|
||||||
|
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
|
||||||
|
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
|
||||||
|
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Never use `git push --tags`, move a published tag, or delete a published tag.
|
||||||
|
|
||||||
|
## Verify The Published Release
|
||||||
|
|
||||||
|
Confirm that the remote tag still points at the guarded commit and that the
|
||||||
|
note is available from the tagged tree:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
|
||||||
|
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
|
||||||
|
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify a fresh source installation and its diagnostic version. The temporary
|
||||||
|
directory confines the installed command to this check:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_verification_dir=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
|
||||||
|
mkdir -p "$release_verification_dir/bin"
|
||||||
|
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
|
||||||
|
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
|
||||||
|
equivalent source verification when local installation policy requires it.
|
||||||
|
`notarius --version` is diagnostic only; downstream compatibility remains
|
||||||
|
defined by the published receipt and artifact contracts.
|
||||||
|
|
||||||
|
## Failure And Correction Policy
|
||||||
|
|
||||||
|
If candidate validation fails before publication, fix the candidate on `main`,
|
||||||
|
rerun the shared checker, and repeat the guards. An unpublished local tag may
|
||||||
|
be deleted after inspection.
|
||||||
|
|
||||||
|
If the remote tag or tag CI reveals a defect, leave the published tag intact.
|
||||||
|
Fix the defect on `main`, choose a new patch version, write a new matching
|
||||||
|
note, and repeat this procedure. Do not weaken tag immutability or add release
|
||||||
|
assets as a workaround.
|
||||||
83
docs/releases/v0.4.0.md
Normal file
83
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# Notarius v0.4.0
|
||||||
|
|
||||||
|
This release strengthens LLM reliability and validation throughout the
|
||||||
|
configured pipeline, upgrades the PromptKit integration, and establishes the
|
||||||
|
source-release and downstream-consumer workflows needed for broader D&D
|
||||||
|
pipeline integration.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Notarius now distinguishes PromptKit structural-output repair from
|
||||||
|
application-owned semantic validation retries. Producer candidates can run
|
||||||
|
through complete deterministic validator chains, receive bounded semantic
|
||||||
|
correction guidance, and retry under explicit stage policies. Final run
|
||||||
|
receipts and manifests preserve bounded validation provenance, while outputs
|
||||||
|
that advance with incomplete validation remain available to the current run
|
||||||
|
without entering reusable checkpoint state.
|
||||||
|
|
||||||
|
The release also adds a maintained complete D&D subprocess-consumer workflow,
|
||||||
|
diagnostic build versions, and the source-only release procedure used to
|
||||||
|
publish this version.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
- Configuration files must use schema version 4. Version 3 is not decoded or
|
||||||
|
rewritten; rename the top-level `scriptorium` section to `promptkit` when
|
||||||
|
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
|
||||||
|
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
|
||||||
|
format and may use its profile-inheritance support. Notarius continues to
|
||||||
|
resolve operator profiles before embedded fallbacks.
|
||||||
|
- Structural-output repair and semantic stage retries are separate bounded
|
||||||
|
mechanisms. Maintained production prompts request one structural repair by
|
||||||
|
default; explicit configuration can override the supported repair count.
|
||||||
|
- Validation policy can now fail a run, reject an output, or permit an
|
||||||
|
otherwise valid candidate to advance with incomplete-validation provenance.
|
||||||
|
The application defaults are documented in
|
||||||
|
[Configuration](../config.md#pipelines).
|
||||||
|
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
|
||||||
|
optional validation summaries plus a required validation-status field.
|
||||||
|
Consumers of this pre-release contract should follow the current
|
||||||
|
[run-result receipt](../integrations/run-result.md).
|
||||||
|
- Existing D&D artifact schema identities remain unchanged. Validation and
|
||||||
|
producer-policy changes can nevertheless cause previously accepted weak
|
||||||
|
candidates to retry, reject, or fail instead.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
|
||||||
|
to `promptkit`.
|
||||||
|
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
|
||||||
|
and ensure their credential environment variables are available at run
|
||||||
|
time.
|
||||||
|
3. Run `notarius config validate --config <path> --pipeline <id>` before the
|
||||||
|
first production invocation.
|
||||||
|
4. Review `structured_output_repair_attempts`, producer retry counts, and
|
||||||
|
`validation_policy` wherever the deployment needs behavior different from
|
||||||
|
the documented defaults.
|
||||||
|
5. Update subprocess consumers to inspect receipt `validation_status` and to
|
||||||
|
tolerate the optional bounded `validation_summaries` field. A consumer that
|
||||||
|
requires fully validated artifacts should require `approved`.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
|
||||||
|
inheritance, structured-output repair, typed error classification, and the
|
||||||
|
correction-aware completion protocol.
|
||||||
|
- Added pipeline and binding configuration for structural repair and terminal
|
||||||
|
validation policy, with strict startup validation and effective-setting
|
||||||
|
provenance.
|
||||||
|
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
|
||||||
|
semantic reconciliation producers. Retry prompts contain the exact defective
|
||||||
|
response and actionable semantic correction guidance without exposing
|
||||||
|
internal reason codes or opaque entity identifiers.
|
||||||
|
- Added complete validator-chain execution, validator retry handling, bounded
|
||||||
|
warnings, terminal dispositions, and durable validation summaries.
|
||||||
|
- Prevented validation-incomplete artifacts and all derived lineage from
|
||||||
|
loading or publishing reusable checkpoints while preserving same-run
|
||||||
|
generated-reference handoff.
|
||||||
|
- Tightened cached chunk-plan validation so only completely validated plans are
|
||||||
|
reused or replace stored plans.
|
||||||
|
- Added a machine-readable subprocess receipt workflow and complete D&D
|
||||||
|
consumer documentation covering all maintained artifacts.
|
||||||
|
- Added source-release checks, immutable lightweight-tag guidance, Linux and
|
||||||
|
Darwin build verification, and diagnostic `notarius --version` output.
|
||||||
@@ -5,13 +5,90 @@ configuration, operations, internal, and integration docs. This roadmap records
|
|||||||
future work only. Items are ordered roughly by current value and specificity,
|
future work only. Items are ordered roughly by current value and specificity,
|
||||||
not as committed release dates.
|
not as committed release dates.
|
||||||
|
|
||||||
|
## Near-Term Validation And LLM Reliability
|
||||||
|
|
||||||
|
PromptKit now owns structural output repair within one completion. Notarius
|
||||||
|
owns stage candidates, validator chains, semantic rejection policy, bounded
|
||||||
|
feedback-aware stage retries, validation provenance, and reusable-state
|
||||||
|
eligibility. The remaining near-term work applies those completed foundations
|
||||||
|
to domain review and operator-facing diagnostics.
|
||||||
|
|
||||||
|
### D&D Combat Scene Semantic Validation
|
||||||
|
|
||||||
|
- Add an optional production LLM-backed D&D validator that determines whether
|
||||||
|
proposed scene boundaries and classifications represent substantive active
|
||||||
|
combat correctly. Its central quality goal is that active combat is kept in
|
||||||
|
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||||
|
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||||
|
- Resolve the validator's exact target before implementation. The current
|
||||||
|
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||||
|
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||||
|
`recap`, and `meta` classification. The preferred initial placement is
|
||||||
|
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||||
|
can compare one proposed kind with the corresponding transcript chunk.
|
||||||
|
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||||
|
question that can be answered from the complete transcript and proposed
|
||||||
|
range map, such as whether one continuous combat was fragmented across
|
||||||
|
inappropriate scene boundaries. Do not duplicate the same classification
|
||||||
|
judgment at both stages. Moving classification into chunk-plan annotations
|
||||||
|
would change the deliberately minimal, annotation-free chunk contract and
|
||||||
|
requires an explicit architecture review before it is selected.
|
||||||
|
- Validate both false negatives and false positives: a non-combat kind must not
|
||||||
|
omit substantive active combat, and a combat kind must be supported by such
|
||||||
|
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||||
|
extraction runs only for an exact `combat` scene classification; semantic
|
||||||
|
review improves the upstream classification but does not replace that gate.
|
||||||
|
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||||
|
structured response schema, and let PromptKit repair structural validator
|
||||||
|
output within its bounded budget. A contract-invalid final validator response
|
||||||
|
is a validator execution failure, not a semantic rejection and not a reason
|
||||||
|
to recursively validate the validator.
|
||||||
|
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||||
|
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||||
|
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||||
|
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||||
|
added calls, latency, and token cost before placing it in the production
|
||||||
|
default chain.
|
||||||
|
- An ADR is not required if classification remains owned by
|
||||||
|
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||||
|
Create or supersede an ADR if the work transfers scene classification into
|
||||||
|
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||||
|
contract.
|
||||||
|
|
||||||
|
### Warning Signal And Presentation Reform
|
||||||
|
|
||||||
|
- Audit every warning producer and representative successful runs. Ordinary
|
||||||
|
success producing dozens of warnings is a failed operator experience: the
|
||||||
|
volume obscures actionable problems and trains operators to ignore the
|
||||||
|
warning channel.
|
||||||
|
- Define a small warning taxonomy that distinguishes actionable degradation,
|
||||||
|
incomplete validation, lossy fallback, and data-quality risk from routine
|
||||||
|
normalization observations or informational diagnostics. Preserve detailed
|
||||||
|
traceability in debug or manifest data without promoting every observation
|
||||||
|
to a top-level CLI warning.
|
||||||
|
- Consider stable deduplication and aggregation by scope and reason code,
|
||||||
|
bounded samples plus omitted counts, and a concise CLI summary with a path to
|
||||||
|
detailed diagnostics. Do not suppress genuine validator execution failures
|
||||||
|
merely to reduce the count.
|
||||||
|
- Decide which warnings affect process status, rejection summaries, durable run
|
||||||
|
receipts, or only debug output. Ensure warning ordering and aggregation are
|
||||||
|
deterministic across concurrent execution.
|
||||||
|
- Establish a representative warning-volume acceptance target and human review
|
||||||
|
workflow before changing individual producers piecemeal. The intended result
|
||||||
|
is not zero warnings; it is a small set in which every surfaced warning merits
|
||||||
|
operator attention.
|
||||||
|
- This work does not require an ADR unless it changes validation acceptance,
|
||||||
|
failure, or durable contract semantics. CLI presentation and diagnostic
|
||||||
|
taxonomy otherwise belong in a feature roadmap followed by updates to their
|
||||||
|
canonical configuration, operations, integration, and internal documents.
|
||||||
|
|
||||||
## Near-Term D&D Pipeline
|
## Near-Term D&D Pipeline
|
||||||
|
|
||||||
### Evaluate Spell Extraction And Normalization
|
### Evaluate Spell Extraction And Normalization
|
||||||
|
|
||||||
- Evaluate ordinary extraction retries and the completed normalization path
|
- Evaluate ordinary extraction retries and the completed normalization path
|
||||||
against a human-reviewed transcript set before adding repair-aware retries or
|
against a human-reviewed transcript set before and after adopting the shared
|
||||||
an LLM-backed semantic validator.
|
PromptKit repair and Notarius validation-retry policies above.
|
||||||
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
||||||
validator, and normalizer development. Treat model-quality review as an
|
validator, and normalizer development. Treat model-quality review as an
|
||||||
iterative human evaluation aid, not a deterministic correctness gate.
|
iterative human evaluation aid, not a deterministic correctness gate.
|
||||||
@@ -28,8 +105,7 @@ The implemented source-backed core and initial D&D registry adoption are
|
|||||||
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
|
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
|
||||||
and
|
and
|
||||||
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||||
The [Semantic Reconciliation Roadmap](semantic-reconciliation.md) retains the
|
The sections below keep broader extensions deferred.
|
||||||
original feature scope; the sections below keep broader extensions deferred.
|
|
||||||
|
|
||||||
### Large-Collection Semantic Reconciliation
|
### Large-Collection Semantic Reconciliation
|
||||||
|
|
||||||
@@ -153,7 +229,6 @@ section only after a concrete workflow, contract, and priority emerge.
|
|||||||
### Distribution And Operations
|
### Distribution And Operations
|
||||||
|
|
||||||
- Packaged release artifacts for alpha distribution.
|
- Packaged release artifacts for alpha distribution.
|
||||||
- A documented versioning and release process.
|
|
||||||
- Optional generated example-output fixtures with a regeneration procedure.
|
- Optional generated example-output fixtures with a regeneration procedure.
|
||||||
- Additional diagnostics or reporting views.
|
- Additional diagnostics or reporting views.
|
||||||
|
|
||||||
|
|||||||
7
go.mod
7
go.mod
@@ -3,9 +3,14 @@ module gitea.maximumdirect.net/eric/notarius
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0
|
gitea.maximumdirect.net/eric/promptkit v0.9.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.40.0
|
require golang.org/x/text v0.40.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -1,5 +1,9 @@
|
|||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
|
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
36
internal/buildinfo/buildinfo.go
Normal file
36
internal/buildinfo/buildinfo.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
// Package buildinfo resolves the product version embedded in a Notarius build.
|
||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"runtime/debug"
|
||||||
|
)
|
||||||
|
|
||||||
|
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
|
||||||
|
|
||||||
|
// Override is set at link time for controlled builds.
|
||||||
|
var Override string
|
||||||
|
|
||||||
|
// Version returns the release version embedded in the build, or development
|
||||||
|
// when the build does not carry a stable release tag.
|
||||||
|
func Version() (string, error) {
|
||||||
|
buildVersion := ""
|
||||||
|
if info, ok := debug.ReadBuildInfo(); ok {
|
||||||
|
buildVersion = info.Main.Version
|
||||||
|
}
|
||||||
|
return resolve(Override, buildVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolve(override, buildVersion string) (string, error) {
|
||||||
|
if override != "" {
|
||||||
|
if !stableVersion.MatchString(override) {
|
||||||
|
return "", fmt.Errorf("build version override is not a stable release tag")
|
||||||
|
}
|
||||||
|
return override, nil
|
||||||
|
}
|
||||||
|
if stableVersion.MatchString(buildVersion) {
|
||||||
|
return buildVersion, nil
|
||||||
|
}
|
||||||
|
return "development", nil
|
||||||
|
}
|
||||||
45
internal/buildinfo/buildinfo_test.go
Normal file
45
internal/buildinfo/buildinfo_test.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestResolve(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
buildVersion string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
|
||||||
|
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
|
||||||
|
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
|
||||||
|
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
|
||||||
|
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
|
||||||
|
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
|
||||||
|
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
|
||||||
|
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
|
||||||
|
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
|
||||||
|
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
|
||||||
|
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
|
||||||
|
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
|
||||||
|
{name: "development build", buildVersion: "(devel)", want: "development"},
|
||||||
|
{name: "missing build information", want: "development"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := resolve(tt.override, tt.buildVersion)
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("resolve() error = nil, want error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("resolve() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,6 +21,10 @@ import (
|
|||||||
|
|
||||||
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||||
|
|
||||||
|
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
|
||||||
|
|
||||||
|
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
|
||||||
|
|
||||||
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||||
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
||||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
@@ -101,6 +105,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
|
||||||
|
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
|
||||||
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||||
|
Prepared: prepared,
|
||||||
|
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||||
|
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
|
||||||
|
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
|
||||||
|
}
|
||||||
|
correction := extractor.correctionSnapshot()
|
||||||
|
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
|
||||||
|
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
||||||
var normalizeChain *pipeline.ResolvedValidatorChain
|
var normalizeChain *pipeline.ResolvedValidatorChain
|
||||||
@@ -136,6 +164,7 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
|||||||
|
|
||||||
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
|
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
|
||||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
||||||
|
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
@@ -161,8 +190,8 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
|
|||||||
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
||||||
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
|
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" || output.Warnings[1].ReasonCode != "spell_not_near_source" {
|
||||||
t.Fatalf("warnings = %#v, want terminal normalize catalog warning", output.Warnings)
|
t.Fatalf("warnings = %#v, want complete terminal normalize validation warnings", output.Warnings)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,6 +235,47 @@ type assembledSpellPipelineOptions struct {
|
|||||||
unknownSpell bool
|
unknownSpell bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
|
||||||
|
t.Helper()
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
extractor := &assembledCorrectingSpellExtractor{}
|
||||||
|
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||||
|
Key: assembledCorrectingSpellExtractorKey,
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||||
|
Requires: []string{"chunks", "source.transcript"},
|
||||||
|
Provides: []string{"dnd.spell_casts"},
|
||||||
|
ArtifactKind: dnd.SpellListKind,
|
||||||
|
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||||
|
return extractor, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register correcting extractor: %v", err)
|
||||||
|
}
|
||||||
|
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
|
||||||
|
return assembledDirectSpellValidator{}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register direct spell validator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
|
||||||
|
extract.Retries = 1
|
||||||
|
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
|
||||||
|
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||||
|
ID: "assembled-dnd-correcting-spells",
|
||||||
|
Input: pipeline.Binding("seriatim"),
|
||||||
|
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
|
||||||
|
},
|
||||||
|
Output: pipeline.Binding("json"),
|
||||||
|
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return components.registries, resolved, extractor
|
||||||
|
}
|
||||||
|
|
||||||
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
@@ -251,6 +321,69 @@ type assembledSpellExtractor struct {
|
|||||||
unknownSpell bool
|
unknownSpell bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type assembledCorrectingSpellExtractor struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
correction *contracts.SemanticCorrection
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
|
||||||
|
|
||||||
|
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
|
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||||
|
if req.Source == nil || req.Chunk == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
|
||||||
|
}
|
||||||
|
response := `{"spell":"accepted"}`
|
||||||
|
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||||
|
if req.Chunk.Index == 0 && req.Correction == nil {
|
||||||
|
response = `{"spell":"Mysterious Burst"}`
|
||||||
|
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||||
|
}
|
||||||
|
if req.Chunk.Index == 0 && req.Correction != nil {
|
||||||
|
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
e.correction = correction
|
||||||
|
e.mu.Unlock()
|
||||||
|
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||||
|
}
|
||||||
|
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||||
|
}
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
correction, err := contracts.CloneSemanticCorrection(e.correction)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return correction
|
||||||
|
}
|
||||||
|
|
||||||
|
type assembledDirectSpellValidator struct{}
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
||||||
|
for _, cast := range req.Value.SpellCasts {
|
||||||
|
if cast.Spell == "Mysterious Burst" {
|
||||||
|
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
||||||
|
|
||||||
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||||
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||||
|
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCommandVersionOutput(t *testing.T) {
|
||||||
|
previous := buildinfo.Override
|
||||||
|
t.Cleanup(func() { buildinfo.Override = previous })
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
wantCode int
|
||||||
|
wantStdout string
|
||||||
|
wantStderr string
|
||||||
|
}{
|
||||||
|
{name: "development", wantStdout: "notarius development\n"},
|
||||||
|
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
|
||||||
|
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buildinfo.Override = tt.override
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
|
||||||
|
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||||
environment := writeCommandConfig(t, "environment", "beta")
|
environment := writeCommandConfig(t, "environment", "beta")
|
||||||
|
|||||||
@@ -189,10 +189,18 @@ func TestMaintainedCompleteExamplePublishesRegistryBackedEntityOccurrences(t *te
|
|||||||
}
|
}
|
||||||
|
|
||||||
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
||||||
for _, laneID := range []string{"enemy-events", "npc-registry", "npc-occurrences", "item-registry", "item-occurrences", "location-registry", "location-occurrences"} {
|
if len(evidence) == 0 {
|
||||||
if !containsString(evidence.SelectedLanes, laneID) || !evidenceHasLane(evidence, laneID) {
|
t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
|
||||||
t.Fatalf("evidence context = %#v, want direct %s evidence", evidence, laneID)
|
|
||||||
}
|
}
|
||||||
|
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
|
||||||
|
for _, unit := range evidence {
|
||||||
|
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||||
|
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
|
||||||
|
}
|
||||||
|
if _, exists := seenEvidenceUnits[unit.ID]; exists {
|
||||||
|
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
|
||||||
|
}
|
||||||
|
seenEvidenceUnits[unit.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
requests := client.requestsFor(enemyevents.PromptID)
|
requests := client.requestsFor(enemyevents.PromptID)
|
||||||
@@ -388,8 +396,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
|
|||||||
if err := json.Unmarshal(content, output); err != nil {
|
if err := json.Unmarshal(content, output); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||||
}
|
}
|
||||||
|
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||||
|
}
|
||||||
client.mu.Lock()
|
client.mu.Lock()
|
||||||
client.requests = append(client.requests, request)
|
client.requests = append(client.requests, snapshot)
|
||||||
client.mu.Unlock()
|
client.mu.Unlock()
|
||||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
|
||||||
}
|
}
|
||||||
@@ -400,7 +412,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
|
|||||||
var requests []contracts.StructuredCompletionRequest
|
var requests []contracts.StructuredCompletionRequest
|
||||||
for _, request := range client.requests {
|
for _, request := range client.requests {
|
||||||
if request.PromptID == promptID {
|
if request.PromptID == promptID {
|
||||||
requests = append(requests, request)
|
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
requests = append(requests, snapshot)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return requests
|
return requests
|
||||||
@@ -415,17 +431,6 @@ func containsString(values []string, want string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func evidenceHasLane(value evidencecontext.Document, laneID string) bool {
|
|
||||||
for _, context := range value.Contexts {
|
|
||||||
for _, reference := range context.EvidenceRefs {
|
|
||||||
if reference.LaneID == laneID {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
||||||
for _, binding := range bindings {
|
for _, binding := range bindings {
|
||||||
if binding.SlotName == slotName && binding.Artifact != nil {
|
if binding.SlotName == slotName && binding.Artifact != nil {
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||||
@@ -331,27 +333,61 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("production prompt engine: %v", err)
|
t.Fatalf("production prompt engine: %v", err)
|
||||||
}
|
}
|
||||||
inputs := map[string]promptkit.ArtifactRef{
|
promptFS, err := components.assets.PromptFS()
|
||||||
"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Alias","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
if err != nil {
|
||||||
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
|
t.Fatalf("production prompt assets: %v", err)
|
||||||
|
}
|
||||||
|
type manifest struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Inputs []struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
} `yaml:"inputs"`
|
||||||
|
Messages []struct {
|
||||||
|
Role string `yaml:"role"`
|
||||||
|
} `yaml:"messages"`
|
||||||
|
}
|
||||||
|
preparedPrompts := 0
|
||||||
|
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(promptFS, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var prompt manifest
|
||||||
|
if err := yaml.Unmarshal(data, &prompt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, message := range prompt.Messages {
|
||||||
|
if message.Role != promptkit.RoleSystem && message.Role != promptkit.RoleUser {
|
||||||
|
return fmt.Errorf("production prompt %q uses role %q, want system or user", prompt.ID, message.Role)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
|
||||||
|
for _, input := range prompt.Inputs {
|
||||||
|
inputs[input.Name] = promptkit.Inline(`{}`)
|
||||||
}
|
}
|
||||||
for _, prompt := range []struct {
|
|
||||||
id string
|
|
||||||
version string
|
|
||||||
}{
|
|
||||||
{id: npcnormalize.PromptID, version: npcnormalize.PromptVersion},
|
|
||||||
{id: itemregistrynormalize.PromptID, version: itemregistrynormalize.PromptVersion},
|
|
||||||
{id: locationnormalize.PromptID, version: locationnormalize.PromptVersion},
|
|
||||||
} {
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: prompt.id, PromptVersion: prompt.version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare production prompt %q: %v", prompt.id, err)
|
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
|
||||||
}
|
}
|
||||||
if prepared.OutputContract.SchemaPath != filepath.Base(semanticreconcile.SchemaAssetPath) {
|
if prepared.OutputContract.RepairAttempts != 1 {
|
||||||
t.Fatalf("prompt %q schema = %q, want generic reconciliation schema", prompt.id, prepared.OutputContract.SchemaPath)
|
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
|
||||||
}
|
}
|
||||||
|
preparedPrompts++
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if preparedPrompts == 0 {
|
||||||
|
t.Fatal("prepared no production prompts")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1063,8 +1099,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
|
|||||||
if err := json.Unmarshal(content, out); err != nil {
|
if err := json.Unmarshal(content, out); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||||
}
|
}
|
||||||
|
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||||
|
}
|
||||||
client.mu.Lock()
|
client.mu.Lock()
|
||||||
client.requests = append(client.requests, req)
|
client.requests = append(client.requests, snapshot)
|
||||||
client.mu.Unlock()
|
client.mu.Unlock()
|
||||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||||
}
|
}
|
||||||
@@ -1075,7 +1115,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
|
|||||||
var requests []contracts.StructuredCompletionRequest
|
var requests []contracts.StructuredCompletionRequest
|
||||||
for _, req := range client.requests {
|
for _, req := range client.requests {
|
||||||
if req.PromptID == promptID {
|
if req.PromptID == promptID {
|
||||||
requests = append(requests, req)
|
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
requests = append(requests, snapshot)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return requests
|
return requests
|
||||||
|
|||||||
@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "malformed profile",
|
name: "malformed profile",
|
||||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
|
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||||
profileID: "malformed-profile",
|
profileID: "malformed-profile",
|
||||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||||
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
|
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid profile source",
|
name: "invalid profile source",
|
||||||
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
|||||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||||
|
for _, profiles := range []string{
|
||||||
|
"id: child\nbase_profile: missing\n",
|
||||||
|
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||||
|
} {
|
||||||
|
t.Run("invalid inheritance", func(t *testing.T) {
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profileID := "child"
|
||||||
|
if strings.Contains(profiles, "id: first") {
|
||||||
|
profileID = "first"
|
||||||
|
}
|
||||||
|
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
|
||||||
|
t.Fatalf("profile preflight error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
@@ -31,6 +32,7 @@ import (
|
|||||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
@@ -62,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
|
if len(args) > 0 && args[0] == "--version" {
|
||||||
|
if len(args) != 1 {
|
||||||
|
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
version, err := buildinfo.Version()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "notarius %s\n", version)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
opts, err = normalizeOptions(opts)
|
opts, err = normalizeOptions(opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -48,8 +48,15 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if info.Mode().Perm() != want {
|
mode := info.Mode().Perm()
|
||||||
t.Fatalf("%s mode = %#o, want %#o", path, info.Mode().Perm(), want)
|
if mode&^want != 0 {
|
||||||
|
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
|
||||||
|
}
|
||||||
|
if info.IsDir() && mode&0o700 != 0o700 {
|
||||||
|
t.Fatalf("%s mode = %#o, want owner access", path, mode)
|
||||||
|
}
|
||||||
|
if !info.IsDir() && mode != want {
|
||||||
|
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,6 +23,7 @@ type runResult struct {
|
|||||||
RejectedOutputCount int `json:"rejected_output_count"`
|
RejectedOutputCount int `json:"rejected_output_count"`
|
||||||
WarningCount int `json:"warning_count"`
|
WarningCount int `json:"warning_count"`
|
||||||
ValidationStatus string `json:"validation_status"`
|
ValidationStatus string `json:"validation_status"`
|
||||||
|
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,6 +61,7 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
|||||||
RejectedOutputCount: len(output.Rejected),
|
RejectedOutputCount: len(output.Rejected),
|
||||||
WarningCount: len(output.Warnings),
|
WarningCount: len(output.Warnings),
|
||||||
ValidationStatus: output.Manifest.ValidationStatus,
|
ValidationStatus: output.Manifest.ValidationStatus,
|
||||||
|
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.TrimSpace(debugDirectory) != "" {
|
if strings.TrimSpace(debugDirectory) != "" {
|
||||||
@@ -85,6 +88,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
||||||
|
if len(summaries) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
||||||
|
for index, summary := range summaries {
|
||||||
|
cloned[index] = artifacts.CloneValidationSummary(summary)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
func encodeRunResult(result runResult) ([]byte, error) {
|
func encodeRunResult(result runResult) ([]byte, error) {
|
||||||
encoded, err := json.Marshal(result)
|
encoded, err := json.Marshal(result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
||||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,6 +55,9 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
|||||||
if got := decoded["warning_count"]; got != float64(1) {
|
if got := decoded["warning_count"]; got != float64(1) {
|
||||||
t.Fatalf("warning_count = %v", got)
|
t.Fatalf("warning_count = %v", got)
|
||||||
}
|
}
|
||||||
|
if got := decoded["validation_summaries"]; got != nil {
|
||||||
|
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
||||||
|
}
|
||||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||||
t.Fatalf("output_directory = %q", got)
|
t.Fatalf("output_directory = %q", got)
|
||||||
}
|
}
|
||||||
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
||||||
|
output := testRunOutput()
|
||||||
|
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
||||||
|
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
||||||
|
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
||||||
|
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
||||||
|
}
|
||||||
|
encoded, err := encodeRunResult(result)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
||||||
|
t.Fatalf("encoded result = %s", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
t.Fatalf("materialize production references: %v", err)
|
t.Fatalf("materialize production references: %v", err)
|
||||||
}
|
}
|
||||||
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
||||||
|
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||||
|
|
||||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||||
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 0 {
|
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "spell_not_near_source" {
|
||||||
t.Fatalf("warnings = %#v, want no emitted warnings from rejected attempts", output.Warnings)
|
t.Fatalf("warnings = %#v, want complete terminal validation warnings", output.Warnings)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -104,6 +104,33 @@ type RejectedOutputManifest struct {
|
|||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
AttemptCount int `json:"attempt_count,omitempty"`
|
AttemptCount int `json:"attempt_count,omitempty"`
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||||
|
Validation *ValidationSummary `json:"validation,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidationSummary is the bounded, durable outcome of validating one
|
||||||
|
// producer result. It deliberately contains identities and stable codes, not
|
||||||
|
// model responses, corrective guidance, validator diagnostics, or payloads.
|
||||||
|
type ValidationSummary struct {
|
||||||
|
Stage string `json:"stage,omitempty"`
|
||||||
|
StepID string `json:"step_id,omitempty"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key,omitempty"`
|
||||||
|
ChunkID string `json:"chunk_id,omitempty"`
|
||||||
|
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
RejectingValidators []string `json:"rejecting_validators,omitempty"`
|
||||||
|
ReasonCodes []string `json:"reason_codes,omitempty"`
|
||||||
|
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
|
||||||
|
ProducerAttemptCount int `json:"producer_attempt_count"`
|
||||||
|
TerminalAction string `json:"terminal_action"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneValidationSummary returns an independently owned durable summary.
|
||||||
|
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
|
||||||
|
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
|
||||||
|
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
|
||||||
|
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
|
||||||
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
type CheckpointDecisionManifest struct {
|
type CheckpointDecisionManifest struct {
|
||||||
@@ -163,6 +190,7 @@ type RunManifest struct {
|
|||||||
References []ReferenceProvenance `json:"references,omitempty"`
|
References []ReferenceProvenance `json:"references,omitempty"`
|
||||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||||
|
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
|
||||||
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
||||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
|||||||
@@ -116,6 +116,11 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
|
|||||||
|
|
||||||
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||||
out := in
|
out := in
|
||||||
|
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
@@ -196,6 +201,11 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
|
|||||||
|
|
||||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||||
out := in
|
out := in
|
||||||
|
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
if len(in.Options) > 0 {
|
if len(in.Options) > 0 {
|
||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
@@ -204,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := *in
|
||||||
|
if in.ProducerStructuralFailure != nil {
|
||||||
|
value := *in.ProducerStructuralFailure
|
||||||
|
out.ProducerStructuralFailure = &value
|
||||||
|
}
|
||||||
|
if in.SemanticRejection != nil {
|
||||||
|
value := *in.SemanticRejection
|
||||||
|
out.SemanticRejection = &value
|
||||||
|
}
|
||||||
|
if in.ValidatorFailure != nil {
|
||||||
|
value := *in.ValidatorFailure
|
||||||
|
out.ValidatorFailure = &value
|
||||||
|
}
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|
||||||
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||||
out := pipeline.ValidatorOverride{Set: in.Set}
|
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||||
if len(in.Validators) > 0 {
|
if len(in.Validators) > 0 {
|
||||||
|
|||||||
@@ -36,6 +36,8 @@ type FilePromptKitLocalBackendConfig struct {
|
|||||||
|
|
||||||
type FilePipelineProfile struct {
|
type FilePipelineProfile struct {
|
||||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
|
||||||
Input fileModuleBinding `yaml:"input"`
|
Input fileModuleBinding `yaml:"input"`
|
||||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||||
@@ -48,15 +50,25 @@ type FilePipelineProfile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||||
|
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
type plainFilePipelineProfile FilePipelineProfile
|
type plainFilePipelineProfile FilePipelineProfile
|
||||||
var decoded plainFilePipelineProfile
|
var decoded plainFilePipelineProfile
|
||||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||||
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
"llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||||
}, "pipeline profile")
|
}, "pipeline profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
*p = FilePipelineProfile(decoded)
|
*p = FilePipelineProfile(decoded)
|
||||||
|
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
|
||||||
|
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.ValidationPolicy = policy
|
||||||
|
}
|
||||||
_, p.artifactsSet = seen["artifacts"]
|
_, p.artifactsSet = seen["artifacts"]
|
||||||
_, p.stepsSet = seen["steps"]
|
_, p.stepsSet = seen["steps"]
|
||||||
_, p.llmProfileSet = seen["llm_profile"]
|
_, p.llmProfileSet = seen["llm_profile"]
|
||||||
@@ -149,6 +161,8 @@ type FileDebugConfig struct {
|
|||||||
type fileModuleBinding struct {
|
type fileModuleBinding struct {
|
||||||
Module string
|
Module string
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
|
ValidationPolicy *pipeline.ValidationPolicyOverride
|
||||||
Retries int
|
Retries int
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
References map[string]fileReferenceSource
|
References map[string]fileReferenceSource
|
||||||
@@ -245,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
b.Module = strings.TrimSpace(module)
|
b.Module = strings.TrimSpace(module)
|
||||||
return nil
|
return nil
|
||||||
case yaml.MappingNode:
|
case yaml.MappingNode:
|
||||||
|
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||||
for i := 0; i < len(node.Content); i += 2 {
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
keyNode := node.Content[i]
|
keyNode := node.Content[i]
|
||||||
valueNode := node.Content[i+1]
|
valueNode := node.Content[i+1]
|
||||||
|
if _, exists := seen[keyNode.Value]; exists {
|
||||||
|
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
|
||||||
|
}
|
||||||
|
seen[keyNode.Value] = struct{}{}
|
||||||
switch keyNode.Value {
|
switch keyNode.Value {
|
||||||
case "module":
|
case "module":
|
||||||
var module string
|
var module string
|
||||||
@@ -264,6 +283,18 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
if b.LLMProfile == "" {
|
if b.LLMProfile == "" {
|
||||||
return fmt.Errorf("llm_profile must not be empty when set")
|
return fmt.Errorf("llm_profile must not be empty when set")
|
||||||
}
|
}
|
||||||
|
case "structured_output_repair_attempts":
|
||||||
|
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.StructuredOutputRepairAttempts = attempts
|
||||||
|
case "validation_policy":
|
||||||
|
policy, err := parseValidationPolicy(valueNode, "module binding")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.ValidationPolicy = policy
|
||||||
case "retries":
|
case "retries":
|
||||||
var retries int
|
var retries int
|
||||||
if err := valueNode.Decode(&retries); err != nil {
|
if err := valueNode.Decode(&retries); err != nil {
|
||||||
@@ -306,13 +337,100 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
return pipeline.ModuleBinding{
|
return pipeline.ModuleBinding{
|
||||||
Module: strings.TrimSpace(b.Module),
|
Module: strings.TrimSpace(b.Module),
|
||||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
|
||||||
Retries: b.Retries,
|
Retries: b.Retries,
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
References: fileReferenceSourcesToPipeline(b.References),
|
References: fileReferenceSourcesToPipeline(b.References),
|
||||||
Validators: b.Validators,
|
Validators: cloneValidatorOverride(b.Validators),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
if node.Content[i].Value == key {
|
||||||
|
return node.Content[i+1], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
|
||||||
|
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy must be an object", context)
|
||||||
|
}
|
||||||
|
policy := &pipeline.ValidationPolicyOverride{}
|
||||||
|
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
key := node.Content[i].Value
|
||||||
|
value := node.Content[i+1]
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
|
||||||
|
}
|
||||||
|
switch key {
|
||||||
|
case "producer_structural_failure":
|
||||||
|
value := pipeline.ProducerStructuralFailureAction(value.Value)
|
||||||
|
policy.ProducerStructuralFailure = &value
|
||||||
|
case "semantic_rejection":
|
||||||
|
value := pipeline.SemanticRejectionAction(value.Value)
|
||||||
|
policy.SemanticRejection = &value
|
||||||
|
case "validator_failure":
|
||||||
|
value := pipeline.ValidatorFailureAction(value.Value)
|
||||||
|
policy.ValidatorFailure = &value
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := policy.Validate(); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
|
||||||
|
}
|
||||||
|
return policy, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
|
||||||
|
if node.Kind != yaml.MappingNode {
|
||||||
|
return fmt.Errorf("%s must be an object", context)
|
||||||
|
}
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
if node.Content[i].Value != "structured_output_repair_attempts" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
|
||||||
|
if node.Tag == "!!null" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
|
||||||
|
}
|
||||||
|
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
|
||||||
|
}
|
||||||
|
var attempts int
|
||||||
|
if err := node.Decode(&attempts); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
|
||||||
|
}
|
||||||
|
if attempts < 0 || attempts > 3 {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||||
|
}
|
||||||
|
return &attempts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
|
||||||
|
if attempts == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value := *attempts
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func LoadFileConfig(path string) (FileConfig, error) {
|
func LoadFileConfig(path string) (FileConfig, error) {
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -520,6 +638,8 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
profile := pipeline.PipelineProfile{
|
profile := pipeline.PipelineProfile{
|
||||||
ID: pipelineID,
|
ID: pipelineID,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
|
||||||
Input: filePipeline.Input.toPipelineBinding(),
|
Input: filePipeline.Input.toPipelineBinding(),
|
||||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||||
|
|||||||
@@ -138,6 +138,168 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
%s
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
validators:
|
||||||
|
- module: validator
|
||||||
|
structured_output_repair_attempts: 3
|
||||||
|
`
|
||||||
|
|
||||||
|
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
|
||||||
|
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := cfg.Pipelines["main"]
|
||||||
|
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["lane"]
|
||||||
|
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
fileProfile := file.Pipelines["main"]
|
||||||
|
fileLane := fileProfile.Artifacts["lane"]
|
||||||
|
*fileLane.Extract.StructuredOutputRepairAttempts = 1
|
||||||
|
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
|
||||||
|
fileProfile.Artifacts["lane"] = fileLane
|
||||||
|
file.Pipelines["main"] = fileProfile
|
||||||
|
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
|
||||||
|
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
|
||||||
|
}
|
||||||
|
cloned := cloneConfig(cfg)
|
||||||
|
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
redacted := cfg.Redacted()
|
||||||
|
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
summary := cfg.RedactedSummaryPayload().(Config)
|
||||||
|
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var roundTripped Config
|
||||||
|
if err := json.Unmarshal(data, &roundTripped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
roundTrippedProfile := roundTripped.Pipelines["main"]
|
||||||
|
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
|
||||||
|
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
%s
|
||||||
|
`
|
||||||
|
const bindingYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input:
|
||||||
|
module: input
|
||||||
|
%s
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
`
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
||||||
const configYAML = `version: 4
|
const configYAML = `version: 4
|
||||||
pipelines:
|
pipelines:
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
|
|||||||
|
|
||||||
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
|
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
|
||||||
out := in
|
out := in
|
||||||
|
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
||||||
out.Input = redactBinding(cloneModuleBinding(in.Input))
|
out.Input = redactBinding(cloneModuleBinding(in.Input))
|
||||||
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
|
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
|
||||||
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||||
|
|||||||
@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
|
||||||
|
semantic := pipeline.SemanticRejectionRejectOutput
|
||||||
|
validator := pipeline.ValidatorFailureFailRun
|
||||||
|
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
|
||||||
|
effective := EffectiveConfig{
|
||||||
|
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
|
||||||
|
"main": {ValidationPolicy: configured},
|
||||||
|
}},
|
||||||
|
ResolvedPipeline: pipeline.ResolvedPipeline{
|
||||||
|
ConfiguredValidationPolicy: configured,
|
||||||
|
ChunkValidationPolicy: pipeline.ValidationPolicy{
|
||||||
|
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
|
||||||
|
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
|
||||||
|
ValidatorFailure: pipeline.ValidatorFailureFailRun,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
|
||||||
|
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
|
||||||
|
}
|
||||||
|
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
|
||||||
|
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
|
||||||
|
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
|
||||||
|
t.Fatal("redacted payload aliases validation policy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func redactionTestBinding(name string) pipeline.ModuleBinding {
|
func redactionTestBinding(name string) pipeline.ModuleBinding {
|
||||||
return pipeline.ModuleBinding{
|
return pipeline.ModuleBinding{
|
||||||
Module: "safe-" + name,
|
Module: "safe-" + name,
|
||||||
|
|||||||
@@ -116,6 +116,14 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
|||||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||||
}
|
}
|
||||||
|
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if profile.ValidationPolicy != nil {
|
||||||
|
if err := profile.ValidationPolicy.Validate(); err != nil {
|
||||||
|
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -195,6 +203,19 @@ func validateBinding(
|
|||||||
binding pipeline.ModuleBinding,
|
binding pipeline.ModuleBinding,
|
||||||
referencesAllowed bool,
|
referencesAllowed bool,
|
||||||
) error {
|
) error {
|
||||||
|
if binding.ValidationPolicy != nil {
|
||||||
|
switch slot {
|
||||||
|
case "chunk", "extract", "merge", "normalize":
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
|
||||||
|
}
|
||||||
|
if err := binding.ValidationPolicy.Validate(); err != nil {
|
||||||
|
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -219,6 +240,13 @@ func validateBinding(
|
|||||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
|
||||||
|
if attempts != nil && (*attempts < 0 || *attempts > 3) {
|
||||||
|
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||||
if !override.Set {
|
if !override.Set {
|
||||||
return nil
|
return nil
|
||||||
@@ -230,6 +258,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
|
|||||||
}
|
}
|
||||||
for i, validator := range override.Validators {
|
for i, validator := range override.Validators {
|
||||||
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||||
|
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if strings.TrimSpace(validator.Module) == "" {
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
return fmt.Errorf("%s module must not be empty", context)
|
return fmt.Errorf("%s module must not be empty", context)
|
||||||
}
|
}
|
||||||
@@ -239,8 +270,11 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
|
|||||||
if validator.Validators.Set {
|
if validator.Validators.Set {
|
||||||
return fmt.Errorf("%s nested validators are not supported", context)
|
return fmt.Errorf("%s nested validators are not supported", context)
|
||||||
}
|
}
|
||||||
if validator.Retries != 0 {
|
if validator.ValidationPolicy != nil {
|
||||||
return fmt.Errorf("%s retries are not supported", context)
|
return fmt.Errorf("%s validation_policy is not supported", context)
|
||||||
|
}
|
||||||
|
if validator.Retries < 0 {
|
||||||
|
return fmt.Errorf("%s retries must be greater than or equal to zero", context)
|
||||||
}
|
}
|
||||||
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
||||||
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
||||||
|
|||||||
@@ -374,17 +374,17 @@ func TestValidateValidatorBindingRules(t *testing.T) {
|
|||||||
want: "chunk validators[0] module must not be empty",
|
want: "chunk validators[0] module must not be empty",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "validator retries",
|
name: "negative validator retries",
|
||||||
setup: func(profile *pipeline.PipelineProfile) {
|
setup: func(profile *pipeline.PipelineProfile) {
|
||||||
profile.Chunk.Validators = pipeline.ValidatorOverride{
|
profile.Chunk.Validators = pipeline.ValidatorOverride{
|
||||||
Set: true,
|
Set: true,
|
||||||
Validators: []pipeline.ModuleBinding{{
|
Validators: []pipeline.ModuleBinding{{
|
||||||
Module: "validator",
|
Module: "validator",
|
||||||
Retries: 1,
|
Retries: -1,
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
want: "chunk validators[0] retries are not supported",
|
want: "chunk validators[0] retries must be greater than or equal to zero",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "validator references",
|
name: "validator references",
|
||||||
|
|||||||
97
internal/core/config/validation_policy_contract_test.go
Normal file
97
internal/core/config/validation_policy_contract_test.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
|
||||||
|
const valid = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
validation_policy:
|
||||||
|
producer_structural_failure: reject_output
|
||||||
|
semantic_rejection: fail_run
|
||||||
|
input: seriatim
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
validation_policy:
|
||||||
|
validator_failure: fail_run
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
validation_policy:
|
||||||
|
semantic_rejection: reject_output
|
||||||
|
`
|
||||||
|
cfg := applyFileConfig(t, valid)
|
||||||
|
profile := cfg.Pipelines["main"]
|
||||||
|
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
|
||||||
|
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
|
||||||
|
}
|
||||||
|
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
|
||||||
|
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["lane"]
|
||||||
|
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
|
||||||
|
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
yaml string
|
||||||
|
}{
|
||||||
|
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
|
||||||
|
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
|
||||||
|
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
|
||||||
|
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
|
||||||
|
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
|
||||||
|
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationPolicyPlacementRules(t *testing.T) {
|
||||||
|
policy := &pipeline.ValidationPolicyOverride{}
|
||||||
|
semantic := pipeline.SemanticRejectionRejectOutput
|
||||||
|
policy.SemanticRejection = &semantic
|
||||||
|
base := pipeline.PipelineProfile{
|
||||||
|
ID: "main",
|
||||||
|
Input: pipeline.Binding("input"),
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"lane": {Extract: pipeline.Binding("extract")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*pipeline.PipelineProfile)
|
||||||
|
}{
|
||||||
|
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
|
||||||
|
{"output", func(profile *pipeline.PipelineProfile) {
|
||||||
|
profile.Output = pipeline.Binding("output")
|
||||||
|
profile.Output.ValidationPolicy = policy
|
||||||
|
}},
|
||||||
|
{"validator", func(profile *pipeline.PipelineProfile) {
|
||||||
|
lane := profile.Artifacts["lane"]
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
|
||||||
|
profile.Artifacts["lane"] = lane
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profile := base
|
||||||
|
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
|
||||||
|
test.mutate(&profile)
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
|
||||||
|
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
@@ -344,7 +345,15 @@ func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.Rejec
|
|||||||
if len(rejected) == 0 {
|
if len(rejected) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
cloned := make([]contracts.RejectedOutput, len(rejected))
|
||||||
|
for index, item := range rejected {
|
||||||
|
cloned[index] = item
|
||||||
|
if item.Validation != nil {
|
||||||
|
summary := artifacts.CloneValidationSummary(*item.Validation)
|
||||||
|
cloned[index].Validation = &summary
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||||
|
|||||||
84
internal/framework/contracts/completion_request_debug.go
Normal file
84
internal/framework/contracts/completion_request_debug.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DebugStructuredCompletionRequest is the content-safe representation of a
|
||||||
|
// structured completion request for ordinary diagnostics and summaries.
|
||||||
|
// Detailed prompt material remains available only through the explicitly
|
||||||
|
// requested LLM debug trace.
|
||||||
|
type DebugStructuredCompletionRequest struct {
|
||||||
|
StageName string `json:"stage_name,omitempty"`
|
||||||
|
PromptID string `json:"prompt_id,omitempty"`
|
||||||
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
InputCount int `json:"input_count"`
|
||||||
|
VariableCount int `json:"variable_count"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Correction *DebugSemanticCorrection `json:"correction,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugSemanticCorrection records only safe correction metadata. It never
|
||||||
|
// exposes the assistant response or user guidance text.
|
||||||
|
type DebugSemanticCorrection struct {
|
||||||
|
AssistantResponseBytes int `json:"assistant_response_bytes"`
|
||||||
|
AssistantResponseDigest string `json:"assistant_response_digest"`
|
||||||
|
UserGuidanceBytes int `json:"user_guidance_bytes"`
|
||||||
|
UserGuidanceDigest string `json:"user_guidance_digest"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugSummary returns a content-safe representation suitable for ordinary
|
||||||
|
// diagnostics. It does not validate or retain correction content.
|
||||||
|
func (request StructuredCompletionRequest) DebugSummary() DebugStructuredCompletionRequest {
|
||||||
|
summary := DebugStructuredCompletionRequest{
|
||||||
|
StageName: request.StageName,
|
||||||
|
PromptID: request.PromptID,
|
||||||
|
PromptVersion: request.PromptVersion,
|
||||||
|
ProfileID: request.ProfileID,
|
||||||
|
SessionID: request.SessionID,
|
||||||
|
InputCount: len(request.Inputs),
|
||||||
|
VariableCount: len(request.Vars),
|
||||||
|
}
|
||||||
|
if request.StructuredOutputRepairAttempts != nil {
|
||||||
|
attempts := *request.StructuredOutputRepairAttempts
|
||||||
|
summary.StructuredOutputRepairAttempts = &attempts
|
||||||
|
}
|
||||||
|
if request.Correction != nil {
|
||||||
|
summary.Correction = request.Correction.DebugSummary()
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugSummary returns content-safe correction metadata suitable for ordinary
|
||||||
|
// diagnostics.
|
||||||
|
func (correction *SemanticCorrection) DebugSummary() *DebugSemanticCorrection {
|
||||||
|
if correction == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &DebugSemanticCorrection{
|
||||||
|
AssistantResponseBytes: len(correction.AssistantResponse),
|
||||||
|
AssistantResponseDigest: debugContentDigest(correction.AssistantResponse),
|
||||||
|
UserGuidanceBytes: len(correction.UserGuidance),
|
||||||
|
UserGuidanceDigest: debugContentDigest([]byte(correction.UserGuidance)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// String prevents ordinary request formatting from exposing correction
|
||||||
|
// content. Use the explicitly requested debug trace for complete messages.
|
||||||
|
func (request StructuredCompletionRequest) String() string {
|
||||||
|
return fmt.Sprintf("%+v", request.DebugSummary())
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString gives %#v formatting the same content-safe behavior as String.
|
||||||
|
func (request StructuredCompletionRequest) GoString() string {
|
||||||
|
return request.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func debugContentDigest(content []byte) string {
|
||||||
|
sum := sha256.Sum256(content)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
@@ -16,6 +16,8 @@ type StructuredCompletionRequest struct {
|
|||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||||
Vars map[string]any `json:"vars,omitempty"`
|
Vars map[string]any `json:"vars,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Correction *SemanticCorrection `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type StructuredCompletionResponse struct {
|
type StructuredCompletionResponse struct {
|
||||||
@@ -26,6 +28,7 @@ type StructuredCompletionResponse struct {
|
|||||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||||
TotalTokens int `json:"total_tokens,omitempty"`
|
TotalTokens int `json:"total_tokens,omitempty"`
|
||||||
|
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||||
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,6 +76,14 @@ type LLMDebugResponse struct {
|
|||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
Validation map[string]any `json:"validation,omitempty"`
|
Validation map[string]any `json:"validation,omitempty"`
|
||||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||||
|
ProviderError *LLMDebugProviderError `json:"provider_error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type LLMDebugProviderError struct {
|
||||||
|
StatusCode int `json:"status_code,omitempty"`
|
||||||
|
Code string `json:"code,omitempty"`
|
||||||
|
Type string `json:"type,omitempty"`
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMDebugUsage struct {
|
type LLMDebugUsage struct {
|
||||||
@@ -130,6 +141,7 @@ type ParseRequest struct {
|
|||||||
Path string `json:"path,omitempty"`
|
Path string `json:"path,omitempty"`
|
||||||
Raw []byte `json:"-"`
|
Raw []byte `json:"-"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,12 +156,15 @@ type ChunkRequest struct {
|
|||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
References ReferenceSet `json:"references,omitempty"`
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Correction *SemanticCorrection `json:"-"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkPlanResult struct {
|
type ChunkPlanResult struct {
|
||||||
Plan source.ChunkPlan `json:"plan"`
|
Plan source.ChunkPlan `json:"plan"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
|
ModelCandidate *ModelCandidate `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Chunker interface {
|
type Chunker interface {
|
||||||
@@ -271,6 +286,7 @@ type ValidationResult struct {
|
|||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code,omitempty"`
|
ReasonCode string `json:"reason_code,omitempty"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
|
CorrectionGuidance string `json:"-"`
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -287,6 +303,7 @@ type OutputRequest struct {
|
|||||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
||||||
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
||||||
@@ -320,6 +337,7 @@ type RejectedOutput struct {
|
|||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
AttemptCount int `json:"attempt_count,omitempty"`
|
AttemptCount int `json:"attempt_count,omitempty"`
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||||
|
Validation *artifacts.ValidationSummary `json:"validation,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ManifestMetadataProvider interface {
|
type ManifestMetadataProvider interface {
|
||||||
|
|||||||
175
internal/framework/contracts/correction.go
Normal file
175
internal/framework/contracts/correction.go
Normal file
@@ -0,0 +1,175 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CorrectionProtocol identifies how a producer can represent the model output
|
||||||
|
// that directly controlled a candidate.
|
||||||
|
type CorrectionProtocol string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CorrectionProtocolSingleResponseV1 CorrectionProtocol = "single_response_v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
MaxValidationReasonCodeBytes = 128
|
||||||
|
MaxValidationCorrectionGuidanceBytes = 4 * 1024
|
||||||
|
MaxAssistantResponseBytes = 1 << 20
|
||||||
|
MaxCorrectionGuidanceBytes = 64 * 1024
|
||||||
|
MaxCorrectionContentBytes = MaxAssistantResponseBytes + MaxCorrectionGuidanceBytes
|
||||||
|
)
|
||||||
|
|
||||||
|
func (protocol CorrectionProtocol) Validate() error {
|
||||||
|
if protocol == CorrectionProtocolSingleResponseV1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("unsupported correction protocol %q", protocol)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SemanticCorrection carries the latest model response and application-owned
|
||||||
|
// guidance for one fresh corrected request. Its content is sensitive and is
|
||||||
|
// deliberately excluded from ordinary JSON serialization.
|
||||||
|
type SemanticCorrection struct {
|
||||||
|
AssistantResponse []byte `json:"-"`
|
||||||
|
UserGuidance string `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewSemanticCorrection(assistantResponse []byte, userGuidance string) (*SemanticCorrection, error) {
|
||||||
|
correction := &SemanticCorrection{
|
||||||
|
AssistantResponse: append([]byte(nil), assistantResponse...),
|
||||||
|
UserGuidance: userGuidance,
|
||||||
|
}
|
||||||
|
if err := correction.Validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return correction, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func CloneSemanticCorrection(correction *SemanticCorrection) (*SemanticCorrection, error) {
|
||||||
|
if correction == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return NewSemanticCorrection(correction.AssistantResponse, correction.UserGuidance)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneStructuredCompletionRequest returns a request whose mutable values are
|
||||||
|
// owned by the caller. It is suitable for clients that retain requests after
|
||||||
|
// CompleteStructured returns.
|
||||||
|
func CloneStructuredCompletionRequest(request StructuredCompletionRequest) (StructuredCompletionRequest, error) {
|
||||||
|
correction, err := CloneSemanticCorrection(request.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return StructuredCompletionRequest{}, fmt.Errorf("clone correction: %w", err)
|
||||||
|
}
|
||||||
|
vars, err := source.CloneMetadata(request.Vars)
|
||||||
|
if err != nil {
|
||||||
|
return StructuredCompletionRequest{}, fmt.Errorf("clone variables: %w", err)
|
||||||
|
}
|
||||||
|
request.Inputs = request.Inputs.Clone()
|
||||||
|
request.Vars = vars
|
||||||
|
request.Correction = correction
|
||||||
|
if request.StructuredOutputRepairAttempts != nil {
|
||||||
|
attempts := *request.StructuredOutputRepairAttempts
|
||||||
|
request.StructuredOutputRepairAttempts = &attempts
|
||||||
|
}
|
||||||
|
return request, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (correction SemanticCorrection) Validate() error {
|
||||||
|
if err := validateAssistantResponse(correction.AssistantResponse); err != nil {
|
||||||
|
return fmt.Errorf("semantic correction assistant response: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateBoundedText(correction.UserGuidance, MaxCorrectionGuidanceBytes, "semantic correction user guidance", false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(correction.AssistantResponse)+len(correction.UserGuidance) > MaxCorrectionContentBytes {
|
||||||
|
return errors.New("semantic correction content exceeds maximum length")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModelCandidate preserves the exact single model response that directly
|
||||||
|
// controlled a producer result. It is attempt-local and never durable data.
|
||||||
|
type ModelCandidate struct {
|
||||||
|
Response []byte `json:"-"`
|
||||||
|
Protocol CorrectionProtocol `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewModelCandidate(response []byte, protocol CorrectionProtocol) (*ModelCandidate, error) {
|
||||||
|
candidate := &ModelCandidate{Response: append([]byte(nil), response...), Protocol: protocol}
|
||||||
|
if err := candidate.Validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return candidate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func CloneModelCandidate(candidate *ModelCandidate) (*ModelCandidate, error) {
|
||||||
|
if candidate == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return NewModelCandidate(candidate.Response, candidate.Protocol)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (candidate ModelCandidate) Validate() error {
|
||||||
|
if err := candidate.Protocol.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateAssistantResponse(candidate.Response); err != nil {
|
||||||
|
return fmt.Errorf("model candidate response: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateValidationResult(result ValidationResult) error {
|
||||||
|
if !result.Approved && result.ReasonCode == "" {
|
||||||
|
return errors.New("validation rejection reason code must not be empty")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != "" {
|
||||||
|
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !result.Approved && result.CorrectionGuidance == "" {
|
||||||
|
return errors.New("validation rejection correction guidance must not be empty")
|
||||||
|
}
|
||||||
|
if result.CorrectionGuidance != "" {
|
||||||
|
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAssistantResponse(response []byte) error {
|
||||||
|
if len(response) > MaxAssistantResponseBytes {
|
||||||
|
return errors.New("exceeds maximum length")
|
||||||
|
}
|
||||||
|
if !utf8.Valid(response) {
|
||||||
|
return errors.New("must be valid UTF-8")
|
||||||
|
}
|
||||||
|
if len(strings.TrimSpace(string(response))) == 0 {
|
||||||
|
return errors.New("must not be blank")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBoundedText(value string, maximum int, name string, optional bool) error {
|
||||||
|
if value == "" && optional {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(value) {
|
||||||
|
return fmt.Errorf("%s must be valid UTF-8", name)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return fmt.Errorf("%s must not be blank", name)
|
||||||
|
}
|
||||||
|
if len(value) > maximum {
|
||||||
|
return fmt.Errorf("%s exceeds maximum length", name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
184
internal/framework/contracts/correction_test.go
Normal file
184
internal/framework/contracts/correction_test.go
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSemanticCorrectionOwnsValidatedContent(t *testing.T) {
|
||||||
|
assistant := []byte(`{"items":["original"]}`)
|
||||||
|
correction, err := NewSemanticCorrection(assistant, "Return one corrected replacement.")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
assistant[0] = '['
|
||||||
|
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||||
|
t.Fatalf("assistant response = %q, want owned original content", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
clone, err := CloneSemanticCorrection(correction)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CloneSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
clone.AssistantResponse[0] = '['
|
||||||
|
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||||
|
t.Fatalf("source correction changed through clone = %q", got)
|
||||||
|
}
|
||||||
|
if nilClone, err := CloneSemanticCorrection(nil); err != nil || nilClone != nil {
|
||||||
|
t.Fatalf("CloneSemanticCorrection(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(correction)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal correction: %v", err)
|
||||||
|
}
|
||||||
|
if string(encoded) != "{}" {
|
||||||
|
t.Fatalf("correction JSON = %s, want no sensitive content", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
|
||||||
|
tooLongAssistant := bytes.Repeat([]byte("a"), MaxAssistantResponseBytes+1)
|
||||||
|
tooLongGuidance := strings.Repeat("a", MaxCorrectionGuidanceBytes+1)
|
||||||
|
tooLongReason := strings.Repeat("a", MaxValidationReasonCodeBytes+1)
|
||||||
|
tooLongValidationGuidance := strings.Repeat("a", MaxValidationCorrectionGuidanceBytes+1)
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
call func() error
|
||||||
|
}{
|
||||||
|
{"blank assistant", func() error { _, err := NewSemanticCorrection([]byte(" \n"), "guidance"); return err }},
|
||||||
|
{"invalid assistant utf8", func() error { _, err := NewSemanticCorrection([]byte{0xff}, "guidance"); return err }},
|
||||||
|
{"oversized assistant", func() error { _, err := NewSemanticCorrection(tooLongAssistant, "guidance"); return err }},
|
||||||
|
{"blank guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), " \t"); return err }},
|
||||||
|
{"invalid guidance utf8", func() error { _, err := NewSemanticCorrection([]byte("response"), string([]byte{0xff})); return err }},
|
||||||
|
{"oversized guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), tooLongGuidance); return err }},
|
||||||
|
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
|
||||||
|
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
|
||||||
|
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
|
||||||
|
{"missing rejection reason code", func() error {
|
||||||
|
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
|
||||||
|
}},
|
||||||
|
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
|
||||||
|
{"oversized reason code", func() error {
|
||||||
|
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
|
||||||
|
}},
|
||||||
|
{"blank reason code", func() error {
|
||||||
|
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
|
||||||
|
}},
|
||||||
|
{"invalid correction guidance utf8", func() error {
|
||||||
|
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
|
||||||
|
}},
|
||||||
|
{"oversized correction guidance", func() error {
|
||||||
|
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if err := test.call(); err == nil {
|
||||||
|
t.Fatal("validation error = nil, want error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModelCandidateOwnsValidatedResponse(t *testing.T) {
|
||||||
|
response := []byte(`{"items":["original"]}`)
|
||||||
|
candidate, err := NewModelCandidate(response, CorrectionProtocolSingleResponseV1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewModelCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
response[0] = '['
|
||||||
|
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||||
|
t.Fatalf("candidate response = %q, want owned original content", got)
|
||||||
|
}
|
||||||
|
clone, err := CloneModelCandidate(candidate)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CloneModelCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
clone.Response[0] = '['
|
||||||
|
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||||
|
t.Fatalf("source candidate changed through clone = %q", got)
|
||||||
|
}
|
||||||
|
if nilClone, err := CloneModelCandidate(nil); err != nil || nilClone != nil {
|
||||||
|
t.Fatalf("CloneModelCandidate(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationResultAllowsAbsentOptionalCorrectionFields(t *testing.T) {
|
||||||
|
if err := ValidateValidationResult(ValidationResult{Approved: true}); err != nil {
|
||||||
|
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := ValidateValidationResult(ValidationResult{ReasonCode: "invalid-evidence", CorrectionGuidance: "Provide source-backed evidence."}); err != nil {
|
||||||
|
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := CorrectionProtocol("").Validate(); err == nil {
|
||||||
|
t.Fatal("empty correction protocol validation error = nil, want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCloneStructuredCompletionRequestOwnsCorrection(t *testing.T) {
|
||||||
|
correction, err := NewSemanticCorrection([]byte(`{"value":"original"}`), "Correct the value.")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
attempts := 2
|
||||||
|
request := StructuredCompletionRequest{
|
||||||
|
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||||
|
Vars: map[string]any{"labels": []string{"original"}},
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
|
Correction: correction,
|
||||||
|
}
|
||||||
|
clone, err := CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CloneStructuredCompletionRequest() error = %v", err)
|
||||||
|
}
|
||||||
|
correction.AssistantResponse[0] = '['
|
||||||
|
request.Inputs["source"] = NewLLMInputMaterial("source", "application/json", []byte(`{"source":false}`), "", "")
|
||||||
|
*request.StructuredOutputRepairAttempts = 7
|
||||||
|
if got := string(clone.Correction.AssistantResponse); got != `{"value":"original"}` {
|
||||||
|
t.Fatalf("cloned correction response = %q, want owned original content", got)
|
||||||
|
}
|
||||||
|
if got := string(clone.Inputs["source"].Content); got != `{"source":true}` {
|
||||||
|
t.Fatalf("cloned input = %q, want owned original content", got)
|
||||||
|
}
|
||||||
|
if clone.StructuredOutputRepairAttempts == nil || *clone.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("cloned repair attempts = %v, want 2", clone.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStructuredCompletionRequestDebugSummaryOmitsCorrectionContent(t *testing.T) {
|
||||||
|
const assistantResponse = `{"secret":"assistant response"}`
|
||||||
|
const userGuidance = "secret user guidance"
|
||||||
|
correction, err := NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
request := StructuredCompletionRequest{
|
||||||
|
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||||
|
Vars: map[string]any{"custom": "value"},
|
||||||
|
Correction: correction,
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := request.DebugSummary()
|
||||||
|
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||||
|
t.Fatalf("debug summary = %#v, want input, variable, and correction metadata", summary)
|
||||||
|
}
|
||||||
|
if summary.Correction.AssistantResponseBytes != len(assistantResponse) || summary.Correction.UserGuidanceBytes != len(userGuidance) ||
|
||||||
|
summary.Correction.AssistantResponseDigest == "" || summary.Correction.UserGuidanceDigest == "" {
|
||||||
|
t.Fatalf("correction summary = %#v, want byte counts and digests", summary.Correction)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal debug summary: %v", err)
|
||||||
|
}
|
||||||
|
for _, rendered := range []string{string(encoded), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request)} {
|
||||||
|
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||||
|
if strings.Contains(rendered, secret) {
|
||||||
|
t.Fatalf("content-safe request rendering leaked %q: %s", secret, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,3 +9,31 @@ var ErrInvalidStructuredOutput = errors.New("invalid structured output")
|
|||||||
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
|
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
|
||||||
// generation begins.
|
// generation begins.
|
||||||
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
|
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
|
||||||
|
|
||||||
|
// ErrLLMGeneration identifies a provider generation failure.
|
||||||
|
var ErrLLMGeneration = errors.New("LLM generation failed")
|
||||||
|
|
||||||
|
type LLMGenerationError struct {
|
||||||
|
status int
|
||||||
|
diagnostic string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
|
||||||
|
if status < 0 {
|
||||||
|
status = 0
|
||||||
|
}
|
||||||
|
return &LLMGenerationError{status: status, diagnostic: diagnostic}
|
||||||
|
}
|
||||||
|
func (e *LLMGenerationError) Error() string {
|
||||||
|
if e == nil || e.diagnostic == "" {
|
||||||
|
return ErrLLMGeneration.Error()
|
||||||
|
}
|
||||||
|
return e.diagnostic
|
||||||
|
}
|
||||||
|
func (e *LLMGenerationError) Unwrap() error { return ErrLLMGeneration }
|
||||||
|
func (e *LLMGenerationError) StatusCode() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.status
|
||||||
|
}
|
||||||
|
|||||||
@@ -41,12 +41,15 @@ type TypedExtractionRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
|
Correction *SemanticCorrection
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedExtractionResult[T any] struct {
|
type TypedExtractionResult[T any] struct {
|
||||||
Value T
|
Value T
|
||||||
Warnings []Warning
|
Warnings []Warning
|
||||||
|
ModelCandidate *ModelCandidate
|
||||||
}
|
}
|
||||||
|
|
||||||
type Extractor[T any] interface {
|
type Extractor[T any] interface {
|
||||||
@@ -63,12 +66,15 @@ type TypedMergeRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
|
Correction *SemanticCorrection
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedMergeResult[T any] struct {
|
type TypedMergeResult[T any] struct {
|
||||||
Value T
|
Value T
|
||||||
Warnings []Warning
|
Warnings []Warning
|
||||||
|
ModelCandidate *ModelCandidate
|
||||||
}
|
}
|
||||||
|
|
||||||
type Merger[T any] interface {
|
type Merger[T any] interface {
|
||||||
@@ -84,6 +90,8 @@ type TypedNormalizeRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
|
Correction *SemanticCorrection
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +99,7 @@ type TypedNormalizeResult[T any] struct {
|
|||||||
Value T
|
Value T
|
||||||
Warnings []Warning
|
Warnings []Warning
|
||||||
Retry *NormalizeRetry
|
Retry *NormalizeRetry
|
||||||
|
ModelCandidate *ModelCandidate
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalize retry diagnostic limits bound module-provided values before the
|
// Normalize retry diagnostic limits bound module-provided values before the
|
||||||
@@ -124,6 +133,7 @@ type TypedValidationRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
@@ -145,6 +155,7 @@ type ChunkValidationRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
}
|
}
|
||||||
@@ -165,6 +176,7 @@ type SerializedValidationRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
|
|||||||
@@ -2,24 +2,8 @@
|
|||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "notarius.source.evidence_context",
|
"$id": "notarius.source.evidence_context",
|
||||||
"title": "notarius_source_evidence_context_v1",
|
"title": "notarius_source_evidence_context_v1",
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["source_id", "source_digest", "window_units", "selected_lanes", "contexts"],
|
|
||||||
"properties": {
|
|
||||||
"source_id": {"type": "string", "minLength": 1},
|
|
||||||
"source_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
|
||||||
"window_units": {"type": "integer", "minimum": 0},
|
|
||||||
"selected_lanes": {
|
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"minItems": 1,
|
"items": {"$ref": "#/$defs/unit"},
|
||||||
"uniqueItems": true,
|
|
||||||
"items": {"type": "string", "minLength": 1}
|
|
||||||
},
|
|
||||||
"contexts": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"$ref": "#/$defs/context"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"$defs": {
|
"$defs": {
|
||||||
"source_ref": {
|
"source_ref": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -42,25 +26,6 @@
|
|||||||
"ref": {"$ref": "#/$defs/source_ref"},
|
"ref": {"$ref": "#/$defs/source_ref"},
|
||||||
"metadata": {"type": "object", "additionalProperties": true}
|
"metadata": {"type": "object", "additionalProperties": true}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"evidence_ref": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["lane_id", "source_ref"],
|
|
||||||
"properties": {
|
|
||||||
"lane_id": {"type": "string", "minLength": 1},
|
|
||||||
"source_ref": {"$ref": "#/$defs/source_ref"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["context_ref", "evidence_refs", "units"],
|
|
||||||
"properties": {
|
|
||||||
"context_ref": {"$ref": "#/$defs/source_ref"},
|
|
||||||
"evidence_refs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evidence_ref"}},
|
|
||||||
"units": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/unit"}}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,118 +3,62 @@ package evidencecontext
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
type contribution struct {
|
|
||||||
laneID string
|
|
||||||
ref source.SourceRef
|
|
||||||
startPos int
|
|
||||||
endPos int
|
|
||||||
}
|
|
||||||
|
|
||||||
type expandedRange struct {
|
type expandedRange struct {
|
||||||
startPos int
|
startPos int
|
||||||
endPos int
|
endPos int
|
||||||
contributions []contribution
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build validates accepted direct references, expands them by source-document
|
// Build validates projected source references, expands them by source-document
|
||||||
// position, and returns their deterministic context union.
|
// position, and returns their ordered union as an owned source-unit excerpt.
|
||||||
func Build(request BuildRequest) (Document, error) {
|
func Build(request BuildRequest) (Document, error) {
|
||||||
if request.WindowUnits < 0 {
|
if request.WindowUnits < 0 {
|
||||||
return Document{}, fmt.Errorf("window_units must not be negative")
|
return nil, fmt.Errorf("window_units must not be negative")
|
||||||
}
|
|
||||||
lanes, err := normalizeSelectedLanes(request.SelectedLanes)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
}
|
||||||
if err := source.ValidateDocument(request.Source); err != nil {
|
if err := source.ValidateDocument(request.Source); err != nil {
|
||||||
return Document{}, fmt.Errorf("validate source document: %w", err)
|
return nil, fmt.Errorf("validate source document: %w", err)
|
||||||
}
|
}
|
||||||
digest, err := source.DigestDocument(request.Source)
|
digest, err := source.DigestDocument(request.Source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, fmt.Errorf("digest source document: %w", err)
|
return nil, fmt.Errorf("digest source document: %w", err)
|
||||||
}
|
}
|
||||||
if digest != request.Source.Digest {
|
if digest != request.Source.Digest {
|
||||||
return Document{}, fmt.Errorf("source digest does not match source document digest")
|
return nil, fmt.Errorf("source digest does not match source document digest")
|
||||||
}
|
}
|
||||||
|
|
||||||
selected := make(map[string]struct{}, len(lanes))
|
|
||||||
for _, laneID := range lanes {
|
|
||||||
selected[laneID] = struct{}{}
|
|
||||||
}
|
|
||||||
index := source.NewDocumentIndex(request.Source)
|
index := source.NewDocumentIndex(request.Source)
|
||||||
seen := make(map[evidenceKey]struct{})
|
ranges := make([]expandedRange, 0, len(request.SourceRefs))
|
||||||
contributions := make([]contribution, 0)
|
for refIndex, ref := range request.SourceRefs {
|
||||||
for laneIndex, laneEvidence := range request.LaneEvidence {
|
|
||||||
laneID := strings.TrimSpace(laneEvidence.LaneID)
|
|
||||||
if _, ok := selected[laneID]; !ok {
|
|
||||||
return Document{}, fmt.Errorf("lane evidence[%d] lane %q is not selected", laneIndex, laneID)
|
|
||||||
}
|
|
||||||
for refIndex, ref := range laneEvidence.SourceRefs {
|
|
||||||
if err := index.ValidateRef(ref); err != nil {
|
if err := index.ValidateRef(ref); err != nil {
|
||||||
return Document{}, fmt.Errorf("lane %q source reference[%d]: %w", laneID, refIndex, err)
|
return nil, fmt.Errorf("source reference[%d]: %w", refIndex, err)
|
||||||
}
|
}
|
||||||
key := evidenceKey{laneID: laneID, ref: ref}
|
|
||||||
if _, exists := seen[key]; exists {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[key] = struct{}{}
|
|
||||||
startPos, _ := index.Position(ref.StartUnitID)
|
startPos, _ := index.Position(ref.StartUnitID)
|
||||||
endPos, _ := index.Position(ref.EndUnitID)
|
endPos, _ := index.Position(ref.EndUnitID)
|
||||||
contributions = append(contributions, contribution{laneID: laneID, ref: ref, startPos: expandStart(startPos, request.WindowUnits), endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits)})
|
ranges = append(ranges, expandedRange{
|
||||||
}
|
startPos: expandStart(startPos, request.WindowUnits),
|
||||||
|
endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(contributions, func(i, j int) bool { return lessContribution(contributions[i], contributions[j]) })
|
merged := mergeRanges(ranges)
|
||||||
document := Document{
|
unitCount := 0
|
||||||
SourceID: request.Source.ID,
|
for _, value := range merged {
|
||||||
SourceDigest: digest,
|
unitCount += value.endPos - value.startPos + 1
|
||||||
WindowUnits: request.WindowUnits,
|
|
||||||
SelectedLanes: lanes,
|
|
||||||
Contexts: make([]Context, 0),
|
|
||||||
}
|
}
|
||||||
for _, rangeValue := range mergeRanges(contributions) {
|
document := make(Document, 0, unitCount)
|
||||||
context, err := buildContext(request.Source, rangeValue)
|
for _, value := range merged {
|
||||||
|
for position := value.startPos; position <= value.endPos; position++ {
|
||||||
|
unit, err := cloneSourceUnit(request.Source.Units[position])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, err
|
return nil, fmt.Errorf("clone source unit at position %d: %w", position, err)
|
||||||
}
|
}
|
||||||
document.Contexts = append(document.Contexts, context)
|
document = append(document, unit)
|
||||||
}
|
}
|
||||||
canonical, err := canonicalizeOwned(document)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, fmt.Errorf("validate evidence context: %w", err)
|
|
||||||
}
|
}
|
||||||
return canonical, nil
|
return document, nil
|
||||||
}
|
|
||||||
|
|
||||||
type evidenceKey struct {
|
|
||||||
laneID string
|
|
||||||
ref source.SourceRef
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizeSelectedLanes(values []string) ([]string, error) {
|
|
||||||
if len(values) == 0 {
|
|
||||||
return nil, fmt.Errorf("selected_lanes must not be empty")
|
|
||||||
}
|
|
||||||
seen := make(map[string]struct{}, len(values))
|
|
||||||
lanes := make([]string, 0, len(values))
|
|
||||||
for index, raw := range values {
|
|
||||||
laneID := strings.TrimSpace(raw)
|
|
||||||
if laneID == "" {
|
|
||||||
return nil, fmt.Errorf("selected_lanes[%d] must not be empty", index)
|
|
||||||
}
|
|
||||||
if _, exists := seen[laneID]; exists {
|
|
||||||
return nil, fmt.Errorf("selected_lanes lane %q is duplicated", laneID)
|
|
||||||
}
|
|
||||||
seen[laneID] = struct{}{}
|
|
||||||
lanes = append(lanes, laneID)
|
|
||||||
}
|
|
||||||
sort.Strings(lanes)
|
|
||||||
return lanes, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func expandStart(position, window int) int {
|
func expandStart(position, window int) int {
|
||||||
@@ -132,65 +76,25 @@ func expandEnd(position, length, window int) int {
|
|||||||
return position + window
|
return position + window
|
||||||
}
|
}
|
||||||
|
|
||||||
func lessContribution(left, right contribution) bool {
|
func mergeRanges(values []expandedRange) []expandedRange {
|
||||||
if left.startPos != right.startPos {
|
|
||||||
return left.startPos < right.startPos
|
|
||||||
}
|
|
||||||
if left.endPos != right.endPos {
|
|
||||||
return left.endPos < right.endPos
|
|
||||||
}
|
|
||||||
return lessEvidenceRef(EvidenceRef{LaneID: left.laneID, SourceRef: left.ref}, EvidenceRef{LaneID: right.laneID, SourceRef: right.ref})
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeRanges(values []contribution) []expandedRange {
|
|
||||||
if len(values) == 0 {
|
if len(values) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
ranges := make([]expandedRange, 0, len(values))
|
sort.Slice(values, func(i, j int) bool {
|
||||||
|
if values[i].startPos != values[j].startPos {
|
||||||
|
return values[i].startPos < values[j].startPos
|
||||||
|
}
|
||||||
|
return values[i].endPos < values[j].endPos
|
||||||
|
})
|
||||||
|
merged := make([]expandedRange, 0, len(values))
|
||||||
for _, value := range values {
|
for _, value := range values {
|
||||||
if len(ranges) == 0 || value.startPos > ranges[len(ranges)-1].endPos+1 {
|
if len(merged) == 0 || value.startPos > merged[len(merged)-1].endPos+1 {
|
||||||
ranges = append(ranges, expandedRange{startPos: value.startPos, endPos: value.endPos, contributions: []contribution{value}})
|
merged = append(merged, value)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
current := &ranges[len(ranges)-1]
|
if value.endPos > merged[len(merged)-1].endPos {
|
||||||
if value.endPos > current.endPos {
|
merged[len(merged)-1].endPos = value.endPos
|
||||||
current.endPos = value.endPos
|
|
||||||
}
|
}
|
||||||
current.contributions = append(current.contributions, value)
|
|
||||||
}
|
}
|
||||||
return ranges
|
return merged
|
||||||
}
|
|
||||||
|
|
||||||
func buildContext(document *source.SourceDocument, value expandedRange) (Context, error) {
|
|
||||||
evidenceRefs := make([]EvidenceRef, 0, len(value.contributions))
|
|
||||||
for _, contribution := range value.contributions {
|
|
||||||
evidenceRefs = append(evidenceRefs, EvidenceRef{LaneID: contribution.laneID, SourceRef: contribution.ref})
|
|
||||||
}
|
|
||||||
sort.Slice(evidenceRefs, func(i, j int) bool { return lessEvidenceRef(evidenceRefs[i], evidenceRefs[j]) })
|
|
||||||
units := make([]source.SourceUnit, 0, value.endPos-value.startPos+1)
|
|
||||||
for position := value.startPos; position <= value.endPos; position++ {
|
|
||||||
unit, err := cloneSourceUnit(document.Units[position])
|
|
||||||
if err != nil {
|
|
||||||
return Context{}, fmt.Errorf("clone source unit at position %d: %w", position, err)
|
|
||||||
}
|
|
||||||
units = append(units, unit)
|
|
||||||
}
|
|
||||||
return Context{
|
|
||||||
ContextRef: source.SourceRef{SourceID: document.ID, StartUnitID: units[0].ID, EndUnitID: units[len(units)-1].ID},
|
|
||||||
EvidenceRefs: evidenceRefs,
|
|
||||||
Units: units,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func lessEvidenceRef(left, right EvidenceRef) bool {
|
|
||||||
if left.LaneID != right.LaneID {
|
|
||||||
return left.LaneID < right.LaneID
|
|
||||||
}
|
|
||||||
if left.SourceRef.SourceID != right.SourceRef.SourceID {
|
|
||||||
return left.SourceRef.SourceID < right.SourceRef.SourceID
|
|
||||||
}
|
|
||||||
if left.SourceRef.StartUnitID != right.SourceRef.StartUnitID {
|
|
||||||
return left.SourceRef.StartUnitID < right.SourceRef.StartUnitID
|
|
||||||
}
|
|
||||||
return left.SourceRef.EndUnitID < right.SourceRef.EndUnitID
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"regexp"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
@@ -18,8 +17,6 @@ import (
|
|||||||
//go:embed assets/schemas/source_evidence_context.v1.json
|
//go:embed assets/schemas/source_evidence_context.v1.json
|
||||||
var schemaAssets embed.FS
|
var schemaAssets embed.FS
|
||||||
|
|
||||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
loadSchemaOnce sync.Once
|
loadSchemaOnce sync.Once
|
||||||
loadedSchema []byte
|
loadedSchema []byte
|
||||||
@@ -78,24 +75,24 @@ func (c *Codec) Encode(value Document) ([]byte, error) {
|
|||||||
|
|
||||||
func (c *Codec) Decode(content []byte) (Document, error) {
|
func (c *Codec) Decode(content []byte) (Document, error) {
|
||||||
if _, err := c.schemaBytes(); err != nil {
|
if _, err := c.schemaBytes(); err != nil {
|
||||||
return Document{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := validateSchemaInstance(content); err != nil {
|
if err := validateSchemaInstance(content); err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
||||||
decoder.DisallowUnknownFields()
|
decoder.DisallowUnknownFields()
|
||||||
var value Document
|
var value Document
|
||||||
if err := decoder.Decode(&value); err != nil {
|
if err := decoder.Decode(&value); err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
var trailing any
|
var trailing any
|
||||||
if err := decoder.Decode(&trailing); err != io.EOF {
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: multiple JSON values")
|
return nil, fmt.Errorf("decode evidence context: multiple JSON values")
|
||||||
}
|
}
|
||||||
canonical, err := canonicalizeOwned(value)
|
canonical, err := canonicalizeOwned(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
return canonical, nil
|
return canonical, nil
|
||||||
}
|
}
|
||||||
@@ -118,14 +115,13 @@ func loadAndCompileSchema() {
|
|||||||
ID string `json:"$id"`
|
ID string `json:"$id"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Required []string `json:"required"`
|
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &identity); err != nil {
|
if err := json.Unmarshal(raw, &identity); err != nil {
|
||||||
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
|
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) {
|
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "array" {
|
||||||
loadSchemaErr = fmt.Errorf("source evidence context schema identity or required fields are invalid")
|
loadSchemaErr = fmt.Errorf("source evidence context schema identity is invalid")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
|
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
|
||||||
@@ -158,164 +154,65 @@ func validateSchemaInstance(content []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func hasRequiredFields(required []string) bool {
|
|
||||||
want := map[string]bool{"source_id": true, "source_digest": true, "window_units": true, "selected_lanes": true, "contexts": true}
|
|
||||||
for _, field := range required {
|
|
||||||
delete(want, field)
|
|
||||||
}
|
|
||||||
return len(want) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalize(value Document) (Document, error) {
|
func canonicalize(value Document) (Document, error) {
|
||||||
owned, err := clone(value)
|
owned, err := clone(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return canonicalizeOwned(owned)
|
return canonicalizeOwned(owned)
|
||||||
}
|
}
|
||||||
|
|
||||||
func canonicalizeOwned(value Document) (Document, error) {
|
func canonicalizeOwned(value Document) (Document, error) {
|
||||||
if err := requireIdentity("source_id", value.SourceID); err != nil {
|
if value == nil {
|
||||||
return Document{}, err
|
return nil, fmt.Errorf("document must be a JSON array")
|
||||||
}
|
}
|
||||||
if !digestPattern.MatchString(value.SourceDigest) {
|
seenUnitIDs := make(map[int]struct{}, len(value))
|
||||||
return Document{}, fmt.Errorf("source_digest must be a sha256 digest")
|
sourceID := ""
|
||||||
|
for unitIndex := range value {
|
||||||
|
unit := value[unitIndex]
|
||||||
|
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
|
||||||
|
return nil, fmt.Errorf("units[%d] has invalid required fields", unitIndex)
|
||||||
}
|
}
|
||||||
if value.WindowUnits < 0 {
|
if err := validateUnitRef(unit, unitIndex); err != nil {
|
||||||
return Document{}, fmt.Errorf("window_units must not be negative")
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := validateSelectedLanes(value.SelectedLanes); err != nil {
|
if sourceID == "" {
|
||||||
return Document{}, err
|
sourceID = unit.Ref.SourceID
|
||||||
|
} else if unit.Ref.SourceID != sourceID {
|
||||||
|
return nil, fmt.Errorf("units[%d].ref.source_id must match units[0].ref.source_id", unitIndex)
|
||||||
}
|
}
|
||||||
if value.Contexts == nil {
|
if _, exists := seenUnitIDs[unit.ID]; exists {
|
||||||
value.Contexts = make([]Context, 0)
|
return nil, fmt.Errorf("units contains duplicate unit id %d", unit.ID)
|
||||||
}
|
}
|
||||||
selected := make(map[string]struct{}, len(value.SelectedLanes))
|
seenUnitIDs[unit.ID] = struct{}{}
|
||||||
for _, laneID := range value.SelectedLanes {
|
|
||||||
selected[laneID] = struct{}{}
|
|
||||||
}
|
|
||||||
seenUnits := make(map[int]struct{})
|
|
||||||
for contextIndex := range value.Contexts {
|
|
||||||
context, err := canonicalizeContext(value.SourceID, selected, seenUnits, value.Contexts[contextIndex], contextIndex)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
|
||||||
value.Contexts[contextIndex] = context
|
|
||||||
}
|
}
|
||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateSelectedLanes(lanes []string) error {
|
func validateUnitRef(unit source.SourceUnit, unitIndex int) error {
|
||||||
if len(lanes) == 0 {
|
prefix := fmt.Sprintf("units[%d].ref", unitIndex)
|
||||||
return fmt.Errorf("selected_lanes must not be empty")
|
if strings.TrimSpace(unit.Ref.SourceID) == "" || strings.TrimSpace(unit.Ref.SourceID) != unit.Ref.SourceID {
|
||||||
}
|
return fmt.Errorf("%s.source_id must be a non-empty trimmed string", prefix)
|
||||||
for index, laneID := range lanes {
|
|
||||||
if err := requireIdentity(fmt.Sprintf("selected_lanes[%d]", index), laneID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if index > 0 && lanes[index-1] >= laneID {
|
|
||||||
return fmt.Errorf("selected_lanes must be unique and in lexical order")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalizeContext(sourceID string, selected map[string]struct{}, seenUnits map[int]struct{}, value Context, contextIndex int) (Context, error) {
|
|
||||||
prefix := fmt.Sprintf("contexts[%d]", contextIndex)
|
|
||||||
if len(value.EvidenceRefs) == 0 {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs must not be empty", prefix)
|
|
||||||
}
|
|
||||||
if len(value.Units) == 0 {
|
|
||||||
return Context{}, fmt.Errorf("%s.units must not be empty", prefix)
|
|
||||||
}
|
|
||||||
if err := validateRefIdentity(sourceID, value.ContextRef, prefix+".context_ref"); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
positions := make(map[int]int, len(value.Units))
|
|
||||||
for unitIndex := range value.Units {
|
|
||||||
unit := value.Units[unitIndex]
|
|
||||||
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
|
|
||||||
return Context{}, fmt.Errorf("%s.units[%d] has invalid required fields", prefix, unitIndex)
|
|
||||||
}
|
|
||||||
if err := validateRefIdentity(sourceID, unit.Ref, fmt.Sprintf("%s.units[%d].ref", prefix, unitIndex)); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
}
|
||||||
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||||
return Context{}, fmt.Errorf("%s.units[%d].ref must identify unit id %d", prefix, unitIndex, unit.ID)
|
return fmt.Errorf("%s must identify unit id %d", prefix, unit.ID)
|
||||||
}
|
|
||||||
if _, exists := positions[unit.ID]; exists {
|
|
||||||
return Context{}, fmt.Errorf("%s.units contains duplicate unit id %d", prefix, unit.ID)
|
|
||||||
}
|
|
||||||
if _, exists := seenUnits[unit.ID]; exists {
|
|
||||||
return Context{}, fmt.Errorf("contexts contain duplicate unit id %d", unit.ID)
|
|
||||||
}
|
|
||||||
positions[unit.ID] = unitIndex
|
|
||||||
seenUnits[unit.ID] = struct{}{}
|
|
||||||
}
|
|
||||||
if value.ContextRef.StartUnitID != value.Units[0].ID || value.ContextRef.EndUnitID != value.Units[len(value.Units)-1].ID {
|
|
||||||
return Context{}, fmt.Errorf("%s.context_ref must identify the first and last units", prefix)
|
|
||||||
}
|
|
||||||
for evidenceIndex := range value.EvidenceRefs {
|
|
||||||
evidence := value.EvidenceRefs[evidenceIndex]
|
|
||||||
if _, ok := selected[evidence.LaneID]; !ok {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs[%d].lane_id is not selected", prefix, evidenceIndex)
|
|
||||||
}
|
|
||||||
if err := requireIdentity(fmt.Sprintf("%s.evidence_refs[%d].lane_id", prefix, evidenceIndex), evidence.LaneID); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
if err := validateRefIdentity(sourceID, evidence.SourceRef, fmt.Sprintf("%s.evidence_refs[%d].source_ref", prefix, evidenceIndex)); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
start, startOK := positions[evidence.SourceRef.StartUnitID]
|
|
||||||
end, endOK := positions[evidence.SourceRef.EndUnitID]
|
|
||||||
if !startOK || !endOK || start > end {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs[%d].source_ref is outside context units", prefix, evidenceIndex)
|
|
||||||
}
|
|
||||||
if evidenceIndex > 0 && !lessEvidenceRef(value.EvidenceRefs[evidenceIndex-1], evidence) {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs must be unique and in deterministic order", prefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateRefIdentity(sourceID string, ref source.SourceRef, field string) error {
|
|
||||||
if ref.SourceID != sourceID {
|
|
||||||
return fmt.Errorf("%s.source_id does not match source_id", field)
|
|
||||||
}
|
|
||||||
if ref.StartUnitID <= 0 || ref.EndUnitID <= 0 {
|
|
||||||
return fmt.Errorf("%s endpoints must be positive", field)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireIdentity(field, value string) error {
|
|
||||||
if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value {
|
|
||||||
return fmt.Errorf("%s must be a non-empty trimmed string", field)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func clone(value Document) (Document, error) {
|
func clone(value Document) (Document, error) {
|
||||||
value.SelectedLanes = append([]string(nil), value.SelectedLanes...)
|
if value == nil {
|
||||||
if value.Contexts == nil {
|
return nil, nil
|
||||||
value.Contexts = make([]Context, 0)
|
}
|
||||||
} else {
|
cloned := make(Document, len(value))
|
||||||
contexts := make([]Context, len(value.Contexts))
|
for unitIndex, unit := range value {
|
||||||
for contextIndex, context := range value.Contexts {
|
owned, err := cloneSourceUnit(unit)
|
||||||
contexts[contextIndex].ContextRef = context.ContextRef
|
|
||||||
contexts[contextIndex].EvidenceRefs = append([]EvidenceRef(nil), context.EvidenceRefs...)
|
|
||||||
contexts[contextIndex].Units = make([]source.SourceUnit, len(context.Units))
|
|
||||||
for unitIndex, unit := range context.Units {
|
|
||||||
cloned, err := cloneSourceUnit(unit)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, fmt.Errorf("clone contexts[%d].units[%d]: %w", contextIndex, unitIndex, err)
|
return nil, fmt.Errorf("clone units[%d]: %w", unitIndex, err)
|
||||||
}
|
}
|
||||||
contexts[contextIndex].Units[unitIndex] = cloned
|
cloned[unitIndex] = owned
|
||||||
}
|
}
|
||||||
}
|
return cloned, nil
|
||||||
value.Contexts = contexts
|
|
||||||
}
|
|
||||||
return value, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
|
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package evidencecontext
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -12,126 +11,57 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildExpandsAndMergesEvidenceByDocumentPosition(t *testing.T) {
|
func TestBuildSelectsExpandedSourceUnitUnion(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
window int
|
window int
|
||||||
evidence []LaneEvidence
|
refs []source.SourceRef
|
||||||
wantUnits [][]int
|
wantIDs []int
|
||||||
wantRefs [][]EvidenceRef
|
|
||||||
}{
|
}{
|
||||||
{
|
{name: "zero window", refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{3}},
|
||||||
name: "zero window",
|
{name: "multi unit citation includes complete range", refs: []source.SourceRef{ref(3, 7)}, wantIDs: []int{3, 30, 7}},
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
{name: "non monotonic IDs use document positions", window: 1, refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{10, 3, 30}},
|
||||||
wantUnits: [][]int{{3}},
|
{name: "boundary clamping", window: 1, refs: []source.SourceRef{ref(10, 10), ref(50, 50)}, wantIDs: []int{10, 3, 7, 50}},
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
|
{name: "overlapping and adjacent windows merge", window: 1, refs: []source.SourceRef{ref(3, 3), ref(30, 30), ref(30, 30)}, wantIDs: []int{10, 3, 30, 7}},
|
||||||
},
|
{name: "adjacent expanded ranges merge", window: 1, refs: []source.SourceRef{ref(10, 10), ref(7, 7)}, wantIDs: []int{10, 3, 30, 7, 50}},
|
||||||
{
|
{name: "largest window clips without overflow", window: math.MaxInt, refs: []source.SourceRef{ref(30, 30)}, wantIDs: []int{10, 3, 30, 7, 50}},
|
||||||
name: "non monotonic ids use positions and clip boundaries",
|
{name: "no references returns an initialized empty document", wantIDs: []int{}},
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "separate gaps stay separate",
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(50, 50)}}},
|
|
||||||
wantUnits: [][]int{{10}, {50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(10, 10)}}, {{LaneID: "npcs", SourceRef: ref(50, 50)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "overlapping windows merge",
|
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}, {LaneID: "npcs", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "contiguous windows merge",
|
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(7, 7)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7, 50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(7, 7)}, {LaneID: "npcs", SourceRef: ref(10, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "duplicate contributions retain unique lane attribution",
|
|
||||||
evidence: []LaneEvidence{
|
|
||||||
{LaneID: "spells", SourceRefs: []source.SourceRef{ref(30, 30), ref(30, 30)}},
|
|
||||||
{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}},
|
|
||||||
},
|
|
||||||
wantUnits: [][]int{{30}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}, {LaneID: "spells", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty contributions retain explicit empty contexts",
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs"}},
|
|
||||||
wantUnits: [][]int{},
|
|
||||||
wantRefs: [][]EvidenceRef{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "largest window clips without overflow",
|
|
||||||
window: math.MaxInt,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7, 50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
document := testDocument(t)
|
got, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: test.window, SourceRefs: test.refs})
|
||||||
got, err := Build(BuildRequest{Source: document, WindowUnits: test.window, SelectedLanes: []string{"spells", "npcs"}, LaneEvidence: test.evidence})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Build() error = %v", err)
|
t.Fatalf("Build() error = %v", err)
|
||||||
}
|
}
|
||||||
if want := []string{"npcs", "spells"}; !reflect.DeepEqual(got.SelectedLanes, want) {
|
if got == nil {
|
||||||
t.Fatalf("SelectedLanes = %#v, want %#v", got.SelectedLanes, want)
|
t.Fatal("Build() returned a nil document")
|
||||||
}
|
}
|
||||||
if got.WindowUnits != test.window || got.SourceID != document.ID || got.SourceDigest != document.Digest {
|
if actual := unitIDs(got); !reflect.DeepEqual(actual, test.wantIDs) {
|
||||||
t.Fatalf("Build() identity = %#v, want source and window identity", got)
|
t.Fatalf("unit IDs = %#v, want %#v", actual, test.wantIDs)
|
||||||
}
|
|
||||||
if actual := contextUnitIDs(got.Contexts); !reflect.DeepEqual(actual, test.wantUnits) {
|
|
||||||
t.Fatalf("context unit ids = %#v, want %#v", actual, test.wantUnits)
|
|
||||||
}
|
|
||||||
if actual := contextEvidenceRefs(got.Contexts); !reflect.DeepEqual(actual, test.wantRefs) {
|
|
||||||
t.Fatalf("context evidence refs = %#v, want %#v", actual, test.wantRefs)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildIsStableAndOwnsSourceAndInputs(t *testing.T) {
|
func TestBuildCopiesSelectedUnitsAndMetadata(t *testing.T) {
|
||||||
document := testDocument(t)
|
document := testDocument(t)
|
||||||
refs := []source.SourceRef{ref(30, 30), ref(3, 3)}
|
first, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
request := BuildRequest{
|
|
||||||
Source: document,
|
|
||||||
WindowUnits: 1,
|
|
||||||
SelectedLanes: []string{"spells", "npcs"},
|
|
||||||
LaneEvidence: []LaneEvidence{{LaneID: "spells", SourceRefs: refs}, {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
|
||||||
}
|
|
||||||
first, err := Build(request)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
secondRequest := request
|
second, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
secondRequest.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}, {LaneID: "spells", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}}
|
|
||||||
second, err := Build(secondRequest)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(first, second) {
|
if !reflect.DeepEqual(first[0], document.Units[0]) {
|
||||||
t.Fatalf("Build() order differs:\nfirst: %#v\nsecond: %#v", first, second)
|
t.Fatalf("first unit = %#v, want unchanged source unit %#v", first[0], document.Units[0])
|
||||||
}
|
}
|
||||||
first.SelectedLanes[0] = "changed"
|
first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
||||||
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Build() returned metadata aliases to source document")
|
t.Fatal("Build() returned metadata aliases to the source document")
|
||||||
}
|
}
|
||||||
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
|
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
|
||||||
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Build() retained metadata aliases to source document")
|
t.Fatal("Build() retained metadata aliases to the source document")
|
||||||
}
|
|
||||||
refs[0].StartUnitID = 999
|
|
||||||
if !containsEvidenceRef(second.Contexts[0].EvidenceRefs, ref(30, 30)) {
|
|
||||||
t.Fatal("Build() retained source-reference input aliases")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,18 +72,11 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
|
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
|
||||||
{name: "blank selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{" "} }, want: "selected_lanes"},
|
|
||||||
{name: "duplicate selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{"npcs", " npcs "} }, want: "duplicated"},
|
|
||||||
{name: "unselected contribution", mutate: func(request *BuildRequest) {
|
|
||||||
request.LaneEvidence = []LaneEvidence{{LaneID: "other", SourceRefs: []source.SourceRef{ref(3, 3)}}}
|
|
||||||
}, want: "not selected"},
|
|
||||||
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
|
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
|
||||||
{name: "invalid reference", mutate: func(request *BuildRequest) {
|
{name: "invalid reference", mutate: func(request *BuildRequest) { request.SourceRefs = []source.SourceRef{ref(99, 99)} }, want: "source reference[0]"},
|
||||||
request.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(99, 99)}}}
|
|
||||||
}, want: "source reference[0]"},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
request := BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}}
|
request := BuildRequest{Source: testDocument(t), SourceRefs: []source.SourceRef{ref(3, 3)}}
|
||||||
test.mutate(&request)
|
test.mutate(&request)
|
||||||
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
|
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
t.Fatalf("Build() error = %v, want %q", err, test.want)
|
t.Fatalf("Build() error = %v, want %q", err, test.want)
|
||||||
@@ -162,7 +85,7 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
func TestCodecRoundTripsFixtureAndOwnsValues(t *testing.T) {
|
||||||
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
|
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -179,15 +102,16 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
|||||||
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
|
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
|
||||||
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
|
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
|
||||||
}
|
}
|
||||||
value.Contexts[0].Units[0].Text = "changed"
|
value[0].Text = "changed"
|
||||||
decoded, err := codec.Decode(encoded)
|
decoded, err := codec.Decode(encoded)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if decoded.Contexts[0].Units[0].Text != "The party meets Rowan." {
|
if decoded[0].Text != "The party meets Rowan." {
|
||||||
t.Fatal("Encode() retained mutable document storage")
|
t.Fatal("Encode() retained mutable document storage")
|
||||||
}
|
}
|
||||||
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
|
|
||||||
|
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -203,82 +127,52 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
||||||
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Decode() returned metadata aliases")
|
t.Fatal("Decode() returned metadata aliases")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) {
|
func TestCodecRejectsInvalidDurablePayloads(t *testing.T) {
|
||||||
value, err := Build(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*Document)
|
content string
|
||||||
}{
|
}{
|
||||||
{name: "unsorted lanes", mutate: func(value *Document) { value.SelectedLanes = []string{"z", "a"} }},
|
{name: "null", content: "null"},
|
||||||
{name: "context range mismatch", mutate: func(value *Document) { value.Contexts[0].ContextRef.EndUnitID = 999 }},
|
{name: "wrapper object", content: `{"units":[]}`},
|
||||||
{name: "mismatched evidence source", mutate: func(value *Document) { value.Contexts[0].EvidenceRefs[0].SourceRef.SourceID = "other" }},
|
{name: "missing required unit field", content: `[{"id":1,"kind":"segment","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
|
||||||
{name: "invalid evidence range", mutate: func(value *Document) {
|
{name: "unknown unit field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1},"unknown":true}]`},
|
||||||
value.Contexts[0].EvidenceRefs[0].SourceRef.StartUnitID = 10
|
{name: "unknown reference field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1,"unknown":true}}]`},
|
||||||
}},
|
{name: "invalid self reference", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":2}}]`},
|
||||||
{name: "duplicate context unit", mutate: func(value *Document) { value.Contexts = append(value.Contexts, value.Contexts[0]) }},
|
{name: "mixed source documents", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session-one","start_unit_id":1,"end_unit_id":1}},{"id":2,"kind":"segment","text":"two","ref":{"source_id":"session-two","start_unit_id":2,"end_unit_id":2}}]`},
|
||||||
|
{name: "duplicate units", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}},{"id":1,"kind":"segment","text":"two","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
|
||||||
|
{name: "multiple JSON values", content: `[] []`},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
candidate, err := clone(value)
|
if _, err := New().Decode([]byte(test.content)); err == nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
test.mutate(&candidate)
|
|
||||||
if _, err := New().Encode(candidate); err == nil {
|
|
||||||
t.Fatal("Encode() error = nil, want durable model rejection")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
content, err := New().Encode(value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
mutate func(map[string]any)
|
|
||||||
}{
|
|
||||||
{name: "missing contexts", mutate: func(value map[string]any) { delete(value, "contexts") }},
|
|
||||||
{name: "null contexts", mutate: func(value map[string]any) { value["contexts"] = nil }},
|
|
||||||
{name: "unknown fixed field", mutate: func(value map[string]any) { value["unknown"] = true }},
|
|
||||||
{name: "missing units", mutate: func(value map[string]any) { delete(contextObject(value, 0), "units") }},
|
|
||||||
{name: "null evidence refs", mutate: func(value map[string]any) { contextObject(value, 0)["evidence_refs"] = nil }},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
raw := decodeJSON(t, content)
|
|
||||||
test.mutate(raw)
|
|
||||||
mutated, err := json.Marshal(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := New().Decode(mutated); err == nil {
|
|
||||||
t.Fatal("Decode() error = nil, want strict payload rejection")
|
t.Fatal("Decode() error = nil, want strict payload rejection")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if _, err := New().Decode(append(content, []byte(" {}")...)); err == nil {
|
if _, err := New().Encode(nil); err == nil {
|
||||||
t.Fatal("Decode() error = nil, want trailing JSON rejection")
|
t.Fatal("Encode(nil) error = nil, want array rejection")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSerializeUsesFixedArtifactIdentity(t *testing.T) {
|
func TestSerializeUsesFixedArtifactIdentityAndEmptyArray(t *testing.T) {
|
||||||
artifact, err := Serialize(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}})
|
artifact, err := Serialize(BuildRequest{Source: testDocument(t)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
|
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
|
||||||
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
|
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
|
||||||
}
|
}
|
||||||
|
if string(artifact.Content) != "[]" {
|
||||||
|
t.Fatalf("Serialize() content = %s, want []", artifact.Content)
|
||||||
|
}
|
||||||
decoded, err := New().Decode(artifact.Content)
|
decoded, err := New().Decode(artifact.Content)
|
||||||
if err != nil || len(decoded.Contexts) != 0 || decoded.Contexts == nil {
|
if err != nil || decoded == nil || len(decoded) != 0 {
|
||||||
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty contexts", decoded, err)
|
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty array", decoded, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,45 +200,10 @@ func ref(start, end int) source.SourceRef {
|
|||||||
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
|
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
|
||||||
}
|
}
|
||||||
|
|
||||||
func contextUnitIDs(contexts []Context) [][]int {
|
func unitIDs(units Document) []int {
|
||||||
values := make([][]int, len(contexts))
|
values := make([]int, len(units))
|
||||||
for index, context := range contexts {
|
for index, unit := range units {
|
||||||
values[index] = make([]int, len(context.Units))
|
values[index] = unit.ID
|
||||||
for unitIndex, unit := range context.Units {
|
|
||||||
values[index][unitIndex] = unit.ID
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return values
|
return values
|
||||||
}
|
}
|
||||||
|
|
||||||
func contextEvidenceRefs(contexts []Context) [][]EvidenceRef {
|
|
||||||
values := make([][]EvidenceRef, len(contexts))
|
|
||||||
for index, context := range contexts {
|
|
||||||
values[index] = append([]EvidenceRef(nil), context.EvidenceRefs...)
|
|
||||||
}
|
|
||||||
return values
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsEvidenceRef(values []EvidenceRef, want source.SourceRef) bool {
|
|
||||||
for _, value := range values {
|
|
||||||
if value.SourceRef == want {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeJSON(t *testing.T, content []byte) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
|
||||||
decoder.UseNumber()
|
|
||||||
var value map[string]any
|
|
||||||
if err := decoder.Decode(&value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
func contextObject(value map[string]any, index int) map[string]any {
|
|
||||||
return value["contexts"].([]any)[index].(map[string]any)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -14,37 +14,12 @@ const (
|
|||||||
MediaType = "application/json"
|
MediaType = "application/json"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Document is the durable union of direct evidence and surrounding source
|
// Document is the durable selected source-unit excerpt.
|
||||||
// context selected for one accepted source document.
|
type Document []source.SourceUnit
|
||||||
type Document struct {
|
|
||||||
SourceID string `json:"source_id"`
|
|
||||||
SourceDigest string `json:"source_digest"`
|
|
||||||
WindowUnits int `json:"window_units"`
|
|
||||||
SelectedLanes []string `json:"selected_lanes"`
|
|
||||||
Contexts []Context `json:"contexts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Context struct {
|
// BuildRequest supplies accepted source material and projected source references.
|
||||||
ContextRef source.SourceRef `json:"context_ref"`
|
|
||||||
EvidenceRefs []EvidenceRef `json:"evidence_refs"`
|
|
||||||
Units []source.SourceUnit `json:"units"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type EvidenceRef struct {
|
|
||||||
LaneID string `json:"lane_id"`
|
|
||||||
SourceRef source.SourceRef `json:"source_ref"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// LaneEvidence attributes direct source references to one selected lane.
|
|
||||||
type LaneEvidence struct {
|
|
||||||
LaneID string `json:"lane_id"`
|
|
||||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildRequest supplies accepted source material and direct lane evidence.
|
|
||||||
type BuildRequest struct {
|
type BuildRequest struct {
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
WindowUnits int
|
WindowUnits int
|
||||||
SelectedLanes []string
|
SourceRefs []source.SourceRef
|
||||||
LaneEvidence []LaneEvidence
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"source_id":"session-alpha","source_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","window_units":0,"selected_lanes":["npcs"],"contexts":[{"context_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13},"evidence_refs":[{"lane_id":"npcs","source_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}],"units":[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]}]}
|
[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -115,6 +116,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
if err := validateOutputTarget(out); err != nil {
|
if err := validateOutputTarget(out); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, err
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
}
|
}
|
||||||
|
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
|
||||||
|
}
|
||||||
|
appendedMessages, err := promptKitCorrectionMessages(req.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion correction: %w", err)
|
||||||
|
}
|
||||||
promptID := strings.TrimSpace(req.PromptID)
|
promptID := strings.TrimSpace(req.PromptID)
|
||||||
if promptID == "" {
|
if promptID == "" {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||||
@@ -136,6 +144,19 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
Inputs: promptKitInputs(req.Inputs),
|
Inputs: promptKitInputs(req.Inputs),
|
||||||
Vars: promptKitVars(req, sessionID),
|
Vars: promptKitVars(req, sessionID),
|
||||||
Execution: execution,
|
Execution: execution,
|
||||||
|
AppendedMessages: appendedMessages,
|
||||||
|
}
|
||||||
|
if req.StructuredOutputRepairAttempts != nil {
|
||||||
|
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
|
||||||
|
if err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("inspect PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||||
|
}
|
||||||
|
contract := inspection.OutputContract
|
||||||
|
contract.RepairAttempts = *req.StructuredOutputRepairAttempts
|
||||||
|
runReq.Validation = &contract
|
||||||
}
|
}
|
||||||
prepared, err := c.engine.PrepareExecution(ctx, runReq)
|
prepared, err := c.engine.PrepareExecution(ctx, runReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -169,6 +190,16 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
redactPromptKitError(err),
|
redactPromptKitError(err),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
status := 0
|
||||||
|
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
|
||||||
|
if errors.As(err, &generationErr) {
|
||||||
|
status = generationErr.StatusCode()
|
||||||
|
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
|
||||||
|
}
|
||||||
|
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
|
||||||
|
}
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||||
}
|
}
|
||||||
if result == nil {
|
if result == nil {
|
||||||
@@ -187,6 +218,41 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
|
||||||
|
owned, err := contracts.CloneSemanticCorrection(correction)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if owned == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []promptkit.RenderedMessage{
|
||||||
|
{Role: promptkit.RoleAssistant, Content: string(owned.AssistantResponse)},
|
||||||
|
{Role: promptkit.RoleUser, Content: owned.UserGuidance},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
|
||||||
|
if generationErr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var secrets []string
|
||||||
|
if prepared != nil && strings.TrimSpace(prepared.EffectiveModelParams.APIKeyEnv) != "" {
|
||||||
|
if value, ok := os.LookupEnv(prepared.EffectiveModelParams.APIKeyEnv); ok {
|
||||||
|
secrets = append(secrets, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
redact := func(value string) string {
|
||||||
|
return bearerTokenPattern.ReplaceAllString(RedactSecrets(value, secrets), "Bearer "+secretReplacement)
|
||||||
|
}
|
||||||
|
return &contracts.LLMDebugResponse{ProviderError: &contracts.LLMDebugProviderError{
|
||||||
|
StatusCode: generationErr.StatusCode(),
|
||||||
|
Code: redact(generationErr.ProviderCode()),
|
||||||
|
Type: redact(generationErr.ProviderType()),
|
||||||
|
Message: redact(generationErr.ProviderMessage()),
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
||||||
content := result.Artifact.Body
|
content := result.Artifact.Body
|
||||||
if len(content) == 0 {
|
if len(content) == 0 {
|
||||||
@@ -210,6 +276,7 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
|
|||||||
PromptTokens: result.Usage.PromptTokens,
|
PromptTokens: result.Usage.PromptTokens,
|
||||||
CompletionTokens: result.Usage.CompletionTokens,
|
CompletionTokens: result.Usage.CompletionTokens,
|
||||||
TotalTokens: result.Usage.TotalTokens,
|
TotalTokens: result.Usage.TotalTokens,
|
||||||
|
RepairAttempts: result.Validation.RepairAttempts,
|
||||||
Debug: promptKitDebugMaterial(prepared, result),
|
Debug: promptKitDebugMaterial(prepared, result),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
package llm
|
package llm
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -115,6 +117,74 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientAppendsSemanticCorrectionAfterRenderedPrompt(t *testing.T) {
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
request := contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.direct-session",
|
||||||
|
ProfileID: "explicit-profile",
|
||||||
|
SessionID: "correction-session",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
Vars: map[string]any{"custom": "value"},
|
||||||
|
}
|
||||||
|
|
||||||
|
var ordinary map[string]any
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), request, &ordinary); err != nil {
|
||||||
|
t.Fatalf("ordinary CompleteStructured() error = %v", err)
|
||||||
|
}
|
||||||
|
ordinaryMessages := append([]promptkit.RenderedMessage(nil), fake.lastRequest().Prompt.Messages...)
|
||||||
|
if len(ordinaryMessages) != 1 {
|
||||||
|
t.Fatalf("ordinary rendered messages = %#v, want only the declared prompt message", ordinaryMessages)
|
||||||
|
}
|
||||||
|
|
||||||
|
correction, err := contracts.NewSemanticCorrection([]byte(`{"previous":"response"}`), "Return the corrected JSON object.")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
request.Correction = correction
|
||||||
|
var corrected map[string]any
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), request, &corrected); err != nil {
|
||||||
|
t.Fatalf("corrected CompleteStructured() error = %v", err)
|
||||||
|
}
|
||||||
|
correctedMessages := fake.lastRequest().Prompt.Messages
|
||||||
|
if len(correctedMessages) != len(ordinaryMessages)+2 {
|
||||||
|
t.Fatalf("corrected message count = %d, want %d", len(correctedMessages), len(ordinaryMessages)+2)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(correctedMessages[:len(ordinaryMessages)], ordinaryMessages) {
|
||||||
|
t.Fatalf("ordinary rendered prefix changed: got %#v, want %#v", correctedMessages[:len(ordinaryMessages)], ordinaryMessages)
|
||||||
|
}
|
||||||
|
if got, want := correctedMessages[len(ordinaryMessages)], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"previous":"response"}`}); got != want {
|
||||||
|
t.Fatalf("assistant correction message = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
if got, want := correctedMessages[len(ordinaryMessages)+1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the corrected JSON object."}); got != want {
|
||||||
|
t.Fatalf("user correction message = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *testing.T) {
|
||||||
|
const sensitiveResponse = "assistant-response-must-not-appear-in-errors"
|
||||||
|
for _, correction := range []*contracts.SemanticCorrection{
|
||||||
|
{AssistantResponse: []byte(sensitiveResponse), UserGuidance: " \t"},
|
||||||
|
{AssistantResponse: bytes.Repeat([]byte(sensitiveResponse), contracts.MaxAssistantResponseBytes/len(sensitiveResponse)+1), UserGuidance: "Use a smaller response."},
|
||||||
|
} {
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
var out map[string]any
|
||||||
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{Correction: correction}, &out)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "correction") {
|
||||||
|
t.Fatalf("CompleteStructured() error = %v, want correction validation failure", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), sensitiveResponse) {
|
||||||
|
t.Fatalf("correction validation error leaked response content: %v", err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 0 {
|
||||||
|
t.Fatalf("provider calls = %d, want no provider call after invalid correction", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
|
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
|
||||||
const initialPrompt = `id: snapshot.test
|
const initialPrompt = `id: snapshot.test
|
||||||
version: "v1"
|
version: "v1"
|
||||||
@@ -456,6 +526,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("inherited parent", func(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||||
|
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||||
|
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||||
@@ -531,6 +627,130 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
|
||||||
|
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
|
||||||
|
t.Fatalf("inspection = %#v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.Header.Get("Authorization"); got != "" {
|
||||||
|
t.Fatalf("Authorization = %q, want omitted", got)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.T) {
|
||||||
|
const credential = "selected-test-credential"
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
_, _ = w.Write([]byte(`{"error":{"code":"temporary","type":"provider_error","message":"marker ` + credential + ` Bearer bearer-secret"}}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "generation-profile", SessionID: "generation-error-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||||
|
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
var generation *contracts.LLMGenerationError
|
||||||
|
if !errors.As(err, &generation) || generation.StatusCode() != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("generation error = %#v", generation)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), credential) || strings.Contains(err.Error(), "marker") {
|
||||||
|
t.Fatalf("ordinary error leaked provider detail: %v", err)
|
||||||
|
}
|
||||||
|
if response.Debug == nil || response.Debug.Response == nil || response.Debug.Response.ProviderError == nil {
|
||||||
|
t.Fatalf("debug = %#v", response.Debug)
|
||||||
|
}
|
||||||
|
debug := response.Debug.Response.ProviderError
|
||||||
|
if debug.StatusCode != http.StatusServiceUnavailable || debug.Code != "temporary" || strings.Contains(debug.Message, credential) || strings.Contains(debug.Message, "bearer-secret") || !strings.Contains(debug.Message, "marker") {
|
||||||
|
t.Fatalf("debug provider error = %#v", debug)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientRejectsInvalidRepairAttemptOverride(t *testing.T) {
|
||||||
|
for _, attempts := range []int{-1, 4} {
|
||||||
|
t.Run("invalid", func(t *testing.T) {
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
var out map[string]any
|
||||||
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", StructuredOutputRepairAttempts: &attempts}, &out)
|
||||||
|
if err == nil || fake.calls != 0 {
|
||||||
|
t.Fatalf("error=%v calls=%d", err, fake.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -772,12 +992,15 @@ func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{{Content: `{"bad":true}`}, {Content: `{"bad":true}`}}}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
|
||||||
var out map[string]any
|
var out map[string]any
|
||||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
PromptID: "adapter.test",
|
PromptID: "adapter.test",
|
||||||
SessionID: "session-123",
|
SessionID: "session-123",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
Inputs: contracts.LLMInputSet{
|
Inputs: contracts.LLMInputSet{
|
||||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
},
|
},
|
||||||
@@ -785,6 +1008,9 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
|||||||
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
||||||
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
||||||
}
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want exhausted repair budget", got)
|
||||||
|
}
|
||||||
if got := string(resp.Content); got != `{"bad":true}` {
|
if got := string(resp.Content); got != `{"bad":true}` {
|
||||||
t.Fatalf("response content = %q, want raw failed output", got)
|
t.Fatalf("response content = %q, want raw failed output", got)
|
||||||
}
|
}
|
||||||
@@ -796,6 +1022,61 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *testing.T) {
|
||||||
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{
|
||||||
|
{Content: `{"bad":true}`, Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8}},
|
||||||
|
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
|
||||||
|
}}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
correction, err := contracts.NewSemanticCorrection([]byte(`{"bad":true}`), "Return the required ok field.")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
}
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
|
SessionID: "repair-test",
|
||||||
|
Correction: correction,
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
if err != nil || !out.OK {
|
||||||
|
t.Fatalf("CompleteStructured() = (%#v, %v), want repaired success", response, err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
|
||||||
|
}
|
||||||
|
requests := fake.requestsSnapshot()
|
||||||
|
if len(requests) != 2 || len(requests[0].Prompt.Messages) < 3 {
|
||||||
|
t.Fatalf("repair requests = %#v, want correction messages on the initial prepared request", requests)
|
||||||
|
}
|
||||||
|
messages := requests[0].Prompt.Messages
|
||||||
|
if got, want := messages[len(messages)-2], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"bad":true}`}); got != want {
|
||||||
|
t.Fatalf("repair assistant correction = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
if got, want := messages[len(messages)-1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the required ok field."}); got != want {
|
||||||
|
t.Fatalf("repair user correction = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
|
||||||
|
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
|
||||||
|
}
|
||||||
|
if response.Debug == nil || response.Debug.Prompt == nil || response.Debug.Response == nil {
|
||||||
|
t.Fatalf("debug = %#v, want prompt and response details", response.Debug)
|
||||||
|
}
|
||||||
|
if response.Debug.Prompt.OutputContract["repair_attempts"] != float64(1) ||
|
||||||
|
response.Debug.Response.Validation["repair_attempts"] != float64(1) ||
|
||||||
|
response.Debug.Response.Content != `{"ok":true}` ||
|
||||||
|
response.Debug.Response.Usage.TotalTokens != 26 {
|
||||||
|
t.Fatalf("debug repair result = %#v, want configured contract and repaired response", response.Debug)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
|
|
||||||
@@ -847,8 +1128,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
|||||||
t.Fatalf("error chain exposes credential: %v", err)
|
t.Fatalf("error chain exposes credential: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errors.Unwrap(err) != nil {
|
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||||
t.Fatalf("provider failure must not expose a wrapped diagnostic: %v", err)
|
t.Fatalf("provider failure = %v, want application generation classification", err)
|
||||||
}
|
}
|
||||||
var recoveredProviderErr *credentialBearingProviderError
|
var recoveredProviderErr *credentialBearingProviderError
|
||||||
if errors.As(err, &recoveredProviderErr) {
|
if errors.As(err, &recoveredProviderErr) {
|
||||||
@@ -857,8 +1138,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
|||||||
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
|
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
|
||||||
}
|
}
|
||||||
if resp.Debug != nil {
|
if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Response != nil {
|
||||||
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
|
t.Fatalf("debug material = %#v, want prepared prompt without provider details", resp.Debug)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -951,6 +1232,15 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
||||||
|
attempts := 1
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
attempts *int
|
||||||
|
}{
|
||||||
|
{name: "during preparation"},
|
||||||
|
{name: "during override inspection", attempts: &attempts},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
@@ -958,6 +1248,7 @@ func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
|||||||
var out map[string]any
|
var out map[string]any
|
||||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
PromptID: "adapter.test",
|
PromptID: "adapter.test",
|
||||||
|
StructuredOutputRepairAttempts: test.attempts,
|
||||||
Inputs: contracts.LLMInputSet{
|
Inputs: contracts.LLMInputSet{
|
||||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
},
|
},
|
||||||
@@ -965,6 +1256,8 @@ func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
|||||||
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||||
}
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
|
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
|
||||||
@@ -1071,6 +1364,66 @@ func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
|||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
|
||||||
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{
|
||||||
|
responses: []promptkit.GenerateResponse{
|
||||||
|
{Content: `{"bad":true}`},
|
||||||
|
{Content: `{"ok":true}`},
|
||||||
|
{Content: `{"ok":true}`},
|
||||||
|
},
|
||||||
|
block: make(chan struct{}),
|
||||||
|
}
|
||||||
|
scheduler, err := NewScheduler(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler() error = %v", err)
|
||||||
|
}
|
||||||
|
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
|
||||||
|
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var out map[string]any
|
||||||
|
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
SessionID: "repair-session",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
firstDone <- callErr
|
||||||
|
}()
|
||||||
|
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||||
|
|
||||||
|
secondDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var out map[string]any
|
||||||
|
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
SessionID: "queued-session",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
secondDone <- callErr
|
||||||
|
}()
|
||||||
|
close(fake.block)
|
||||||
|
if err := <-firstDone; err != nil {
|
||||||
|
t.Fatalf("repaired completion error = %v", err)
|
||||||
|
}
|
||||||
|
if err := <-secondDone; err != nil {
|
||||||
|
t.Fatalf("queued completion error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
requests := fake.requestsSnapshot()
|
||||||
|
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
|
||||||
|
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||||
|
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
var out map[string]any
|
var out map[string]any
|
||||||
@@ -1225,11 +1578,13 @@ func (f *switchingPromptFS) resumePromptRead() {
|
|||||||
|
|
||||||
type fakePromptKitLLM struct {
|
type fakePromptKitLLM struct {
|
||||||
content string
|
content string
|
||||||
|
responses []promptkit.GenerateResponse
|
||||||
allowEmpty bool
|
allowEmpty bool
|
||||||
err error
|
err error
|
||||||
block chan struct{}
|
block chan struct{}
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
last promptkit.GenerateRequest
|
last promptkit.GenerateRequest
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
calls int32
|
calls int32
|
||||||
inFlight int32
|
inFlight int32
|
||||||
maxInFlight int32
|
maxInFlight int32
|
||||||
@@ -1244,8 +1599,9 @@ func (*credentialBearingProviderError) Error() string {
|
|||||||
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
f.last = req
|
f.last = req
|
||||||
|
f.requests = append(f.requests, req)
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
atomic.AddInt32(&f.calls, 1)
|
call := atomic.AddInt32(&f.calls, 1)
|
||||||
current := atomic.AddInt32(&f.inFlight, 1)
|
current := atomic.AddInt32(&f.inFlight, 1)
|
||||||
for {
|
for {
|
||||||
seen := atomic.LoadInt32(&f.maxInFlight)
|
seen := atomic.LoadInt32(&f.maxInFlight)
|
||||||
@@ -1264,6 +1620,14 @@ func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateR
|
|||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, f.err
|
||||||
}
|
}
|
||||||
|
if len(f.responses) > 0 {
|
||||||
|
index := int(call - 1)
|
||||||
|
if index >= len(f.responses) {
|
||||||
|
return nil, fmt.Errorf("unexpected provider call %d", call)
|
||||||
|
}
|
||||||
|
response := f.responses[index]
|
||||||
|
return &response, nil
|
||||||
|
}
|
||||||
content := f.content
|
content := f.content
|
||||||
if content == "" && !f.allowEmpty {
|
if content == "" && !f.allowEmpty {
|
||||||
content = `{"ok":true}`
|
content = `{"ok":true}`
|
||||||
@@ -1294,3 +1658,9 @@ func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
|
|||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
return f.last
|
return f.last
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptKitLLM) requestsSnapshot() []promptkit.GenerateRequest {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return append([]promptkit.GenerateRequest(nil), f.requests...)
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const (
|
|||||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||||
// release. Update this identity when the dependency is upgraded.
|
// release. Update this identity when the dependency is upgraded.
|
||||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
|
promptKitBuiltinProfileCatalogID = "promptkit:v0.9.0:builtin-profiles"
|
||||||
)
|
)
|
||||||
|
|
||||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ const (
|
|||||||
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
|
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
|
||||||
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
|
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
|
||||||
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
|
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
|
||||||
|
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CheckpointDecision struct {
|
type CheckpointDecision struct {
|
||||||
@@ -160,6 +161,8 @@ func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
|
|||||||
return "accepted normalized artifact is reusable"
|
return "accepted normalized artifact is reusable"
|
||||||
case CheckpointReasonRecomputeStep:
|
case CheckpointReasonRecomputeStep:
|
||||||
return "selected step requires execution"
|
return "selected step requires execution"
|
||||||
|
case CheckpointReasonValidationIncompleteLineage:
|
||||||
|
return "checkpoint reuse is disabled by validation-incomplete input lineage"
|
||||||
default:
|
default:
|
||||||
return "checkpoint decision"
|
return "checkpoint decision"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ type debugTimedEnvelope struct {
|
|||||||
LaneID string `json:"lane_id,omitempty"`
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key,omitempty"`
|
ModuleKey string `json:"module_key,omitempty"`
|
||||||
Attempt int `json:"attempt,omitempty"`
|
Attempt int `json:"attempt,omitempty"`
|
||||||
|
AttemptKind string `json:"attempt_kind,omitempty"`
|
||||||
StartedAt time.Time `json:"started_at"`
|
StartedAt time.Time `json:"started_at"`
|
||||||
CompletedAt time.Time `json:"completed_at"`
|
CompletedAt time.Time `json:"completed_at"`
|
||||||
DurationMS int64 `json:"duration_ms"`
|
DurationMS int64 `json:"duration_ms"`
|
||||||
@@ -110,24 +111,7 @@ type debugSerializedOutput struct {
|
|||||||
Content debugBinaryEnvelope `json:"content"`
|
Content debugBinaryEnvelope `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type debugLLMInputMaterial struct {
|
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
|
||||||
Name string `json:"name"`
|
|
||||||
MediaType string `json:"media_type,omitempty"`
|
|
||||||
Content string `json:"content_base64,omitempty"`
|
|
||||||
Digest string `json:"digest,omitempty"`
|
|
||||||
OriginURI string `json:"origin_uri,omitempty"`
|
|
||||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type debugStructuredCompletionRequest struct {
|
|
||||||
StageName string `json:"stage_name"`
|
|
||||||
PromptID string `json:"prompt_id,omitempty"`
|
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
|
|
||||||
Vars map[string]any `json:"vars,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type debugStructuredCompletionResponse struct {
|
type debugStructuredCompletionResponse struct {
|
||||||
Content string `json:"content,omitempty"`
|
Content string `json:"content,omitempty"`
|
||||||
@@ -160,9 +144,37 @@ type debugLLMCallReference struct {
|
|||||||
PromptID string `json:"prompt_id,omitempty"`
|
PromptID string `json:"prompt_id,omitempty"`
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
|
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||||
|
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||||
|
TotalTokens int `json:"total_tokens,omitempty"`
|
||||||
|
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||||
Error bool `json:"error,omitempty"`
|
Error bool `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type debugValidationOutcome struct {
|
||||||
|
ValidatorName string `json:"validator_name"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
AttemptCount int `json:"attempt_count"`
|
||||||
|
ReasonCode string `json:"reason_code,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type debugProducerAttempt struct {
|
||||||
|
Number int `json:"number"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
Validation []debugValidationOutcome `json:"validation,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type debugProducerTerminal struct {
|
||||||
|
ProducerAttemptCount int `json:"producer_attempt_count"`
|
||||||
|
Attempts []debugProducerAttempt `json:"attempts,omitempty"`
|
||||||
|
AggregateReasonCodes []string `json:"aggregate_reason_codes,omitempty"`
|
||||||
|
ValidationComplete bool `json:"validation_complete"`
|
||||||
|
EffectivePolicy ValidationPolicy `json:"effective_policy"`
|
||||||
|
TerminalAction string `json:"terminal_action"`
|
||||||
|
Summary artifacts.ValidationSummary `json:"validation_summary"`
|
||||||
|
}
|
||||||
|
|
||||||
type debugValidationCall struct {
|
type debugValidationCall struct {
|
||||||
ValidatorName string `json:"validator_name"`
|
ValidatorName string `json:"validator_name"`
|
||||||
Request any `json:"request"`
|
Request any `json:"request"`
|
||||||
@@ -267,6 +279,10 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
|
|||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
|
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
|
||||||
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
|
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
|
||||||
|
PromptTokens: response.PromptTokens,
|
||||||
|
CompletionTokens: response.CompletionTokens,
|
||||||
|
TotalTokens: response.TotalTokens,
|
||||||
|
RepairAttempts: response.RepairAttempts,
|
||||||
Error: err != nil,
|
Error: err != nil,
|
||||||
}
|
}
|
||||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||||
@@ -423,6 +439,50 @@ func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
|
|||||||
return terminalErr
|
return terminalErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func debugValidationOutcomes(report validationReport) []debugValidationOutcome {
|
||||||
|
if len(report.records) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
outcomes := make([]debugValidationOutcome, 0, len(report.records))
|
||||||
|
for _, record := range report.records {
|
||||||
|
outcomes = append(outcomes, debugValidationOutcome{
|
||||||
|
ValidatorName: record.validatorName,
|
||||||
|
Outcome: string(record.outcome),
|
||||||
|
AttemptCount: record.attemptCount,
|
||||||
|
ReasonCode: record.reasonCode,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return outcomes
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeProducerTerminalDebug(recorder DebugRecorder, name string, terminal producerAttemptTerminal, policy ValidationPolicy, summary artifacts.ValidationSummary) error {
|
||||||
|
attempts := make([]debugProducerAttempt, 0, len(terminal.Provenance))
|
||||||
|
for _, item := range terminal.Provenance {
|
||||||
|
attempts = append(attempts, debugProducerAttempt{
|
||||||
|
Number: item.Number,
|
||||||
|
Kind: string(item.Kind),
|
||||||
|
Outcome: string(item.Outcome),
|
||||||
|
Validation: debugValidationOutcomes(item.Validation),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return writeDebugTimed(recorder, name, debugTimedEnvelope{
|
||||||
|
Stage: summary.Stage,
|
||||||
|
StepID: summary.StepID,
|
||||||
|
LaneID: summary.LaneID,
|
||||||
|
ModuleKey: summary.ModuleKey,
|
||||||
|
StartedAt: time.Now().UTC(),
|
||||||
|
Payload: debugProducerTerminal{
|
||||||
|
ProducerAttemptCount: summary.ProducerAttemptCount,
|
||||||
|
Attempts: attempts,
|
||||||
|
AggregateReasonCodes: append([]string(nil), summary.ReasonCodes...),
|
||||||
|
ValidationComplete: summary.Status == "complete",
|
||||||
|
EffectivePolicy: policy,
|
||||||
|
TerminalAction: string(terminal.Action),
|
||||||
|
Summary: artifacts.CloneValidationSummary(summary),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
|
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
|
||||||
content = redactSecretBytes(content)
|
content = redactSecretBytes(content)
|
||||||
return debugBinaryEnvelope{
|
return debugBinaryEnvelope{
|
||||||
@@ -567,29 +627,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
||||||
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
|
return req.DebugSummary()
|
||||||
for key, material := range req.Inputs {
|
|
||||||
inputs[key] = debugLLMInputMaterial{
|
|
||||||
Name: material.Name,
|
|
||||||
MediaType: material.MediaType,
|
|
||||||
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
|
|
||||||
Digest: material.Digest,
|
|
||||||
OriginURI: material.OriginURI,
|
|
||||||
SizeBytes: material.SizeBytes,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(inputs) == 0 {
|
|
||||||
inputs = nil
|
|
||||||
}
|
|
||||||
return debugStructuredCompletionRequest{
|
|
||||||
StageName: req.StageName,
|
|
||||||
PromptID: req.PromptID,
|
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
|
||||||
SessionID: req.SessionID,
|
|
||||||
Inputs: inputs,
|
|
||||||
Vars: redactSensitiveMap(req.Vars),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
||||||
@@ -691,6 +729,7 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
|
|||||||
|
|
||||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||||
|
result.CorrectionGuidance = ""
|
||||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||||
for i := range result.Warnings {
|
for i := range result.Warnings {
|
||||||
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
|
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
|
||||||
|
|||||||
@@ -47,6 +47,32 @@ func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) {
|
||||||
|
const assistantResponse = `{"secret":"assistant response"}`
|
||||||
|
const userGuidance = "secret user guidance"
|
||||||
|
correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||||
|
}
|
||||||
|
summary := debugCompletionRequest(contracts.StructuredCompletionRequest{
|
||||||
|
Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||||
|
Vars: map[string]any{"custom": "value"},
|
||||||
|
Correction: correction,
|
||||||
|
})
|
||||||
|
encoded, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal completion summary: %v", err)
|
||||||
|
}
|
||||||
|
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||||
|
if strings.Contains(string(encoded), secret) {
|
||||||
|
t.Fatalf("debug completion summary leaked %q: %s", secret, encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||||
|
t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
|
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
|
||||||
doc := validSourceDocument()
|
doc := validSourceDocument()
|
||||||
envelope := debugSourceDocumentEnvelope(doc)
|
envelope := debugSourceDocumentEnvelope(doc)
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ type debugEvidenceContextSummary struct {
|
|||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
SelectedLanes []string `json:"selected_lanes"`
|
SelectedLanes []string `json:"selected_lanes"`
|
||||||
WindowUnits int `json:"window_units"`
|
WindowUnits int `json:"window_units"`
|
||||||
ContextCount int `json:"context_count"`
|
|
||||||
UnitCount int `json:"unit_count"`
|
UnitCount int `json:"unit_count"`
|
||||||
SourceDigest string `json:"source_digest"`
|
SourceDigest string `json:"source_digest"`
|
||||||
}
|
}
|
||||||
@@ -51,8 +50,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
request := evidencecontext.BuildRequest{
|
request := evidencecontext.BuildRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
||||||
SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...),
|
SourceRefs: make([]source.SourceRef, 0),
|
||||||
LaneEvidence: make([]evidencecontext.LaneEvidence, 0, len(prepared.evidencePlan.lanes)),
|
|
||||||
}
|
}
|
||||||
for _, lane := range prepared.evidencePlan.lanes {
|
for _, lane := range prepared.evidencePlan.lanes {
|
||||||
output, ok := byLane[lane.laneID]
|
output, ok := byLane[lane.laneID]
|
||||||
@@ -73,10 +71,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID)
|
return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID)
|
||||||
}
|
}
|
||||||
request.LaneEvidence = append(request.LaneEvidence, evidencecontext.LaneEvidence{
|
request.SourceRefs = append(request.SourceRefs, references...)
|
||||||
LaneID: lane.laneID,
|
|
||||||
SourceRefs: append([]source.SourceRef(nil), references...),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
document, err := evidencecontext.Build(request)
|
document, err := evidencecontext.Build(request)
|
||||||
@@ -99,13 +94,10 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
SchemaID: artifact.Schema.ID,
|
SchemaID: artifact.Schema.ID,
|
||||||
SchemaName: artifact.Schema.Name,
|
SchemaName: artifact.Schema.Name,
|
||||||
SchemaVersion: artifact.Schema.Version,
|
SchemaVersion: artifact.Schema.Version,
|
||||||
SelectedLanes: append([]string(nil), document.SelectedLanes...),
|
SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...),
|
||||||
WindowUnits: document.WindowUnits,
|
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
||||||
ContextCount: len(document.Contexts),
|
SourceDigest: doc.Digest,
|
||||||
SourceDigest: document.SourceDigest,
|
UnitCount: len(document),
|
||||||
}
|
|
||||||
for _, context := range document.Contexts {
|
|
||||||
summary.UnitCount += len(context.Units)
|
|
||||||
}
|
}
|
||||||
return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil
|
return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,14 +97,11 @@ func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if !reflect.DeepEqual(value.SelectedLanes, []string{"alpha", "beta", "inactive"}) || len(value.Contexts) != 1 || len(value.Contexts[0].Units) != 3 {
|
if actual := []int{value[0].ID, value[1].ID, value[2].ID}; !reflect.DeepEqual(actual, []int{1, 2, 3}) {
|
||||||
t.Fatalf("evidence context = %#v, want selected union", value)
|
t.Fatalf("evidence context = %#v, want selected union", value)
|
||||||
}
|
}
|
||||||
if got := value.Contexts[0].EvidenceRefs; len(got) != 2 || got[0].LaneID != "alpha" || got[1].LaneID != "beta" {
|
|
||||||
t.Fatalf("evidence refs = %#v, want both selected lanes", got)
|
|
||||||
}
|
|
||||||
debugJSON := string(debug.json["output/evidence-context.json"])
|
debugJSON := string(debug.json["output/evidence-context.json"])
|
||||||
if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || !strings.Contains(debugJSON, `"context_count":1`) || !strings.Contains(debugJSON, `"unit_count":3`) {
|
if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || strings.Contains(debugJSON, "context_count") || !strings.Contains(debugJSON, `"unit_count":3`) {
|
||||||
t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON)
|
t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -117,9 +114,10 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
|
|||||||
prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{
|
prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||||
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
|
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
|
prepared.Steps[1].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
installEvidencePlan(prepared, 0, []string{"absent", "present", "rejected"}, func(notes codecNotes) ([]source.SourceRef, error) {
|
installEvidencePlan(prepared, 0, []string{"absent", "present", "rejected"}, func(notes codecNotes) ([]source.SourceRef, error) {
|
||||||
if len(notes.Items) > 0 && notes.Items[0] == "present" {
|
if len(notes.Items) > 0 && notes.Items[0] == "present" {
|
||||||
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil
|
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil
|
||||||
@@ -134,7 +132,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
|
|||||||
t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected)
|
t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected)
|
||||||
}
|
}
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if len(value.Contexts) != 1 || len(value.Contexts[0].EvidenceRefs) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "present" {
|
if len(value) != 1 || value[0].ID != 1 {
|
||||||
t.Fatalf("evidence context = %#v, want present lane only", value)
|
t.Fatalf("evidence context = %#v, want present lane only", value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -232,7 +230,7 @@ func TestRunnerEvidenceContextRebuildsFromAcceptedCheckpoint(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if len(value.Contexts) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "notes" {
|
if len(value) != 1 || value[0].ID != 1 {
|
||||||
t.Fatalf("evidence context = %#v, want checkpointed normalized output", value)
|
t.Fatalf("evidence context = %#v, want checkpointed normalized output", value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,11 +58,20 @@ func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpe
|
|||||||
if !ok {
|
if !ok {
|
||||||
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
||||||
}
|
}
|
||||||
|
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone extraction correction: %w", err)
|
||||||
|
}
|
||||||
|
request.Correction = correction
|
||||||
result, err := extractor.Extract(ctx, request)
|
result, err := extractor.Extract(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
|
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone extraction model candidate: %w", err)
|
||||||
|
}
|
||||||
|
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), ModelCandidate: candidate}, nil
|
||||||
}}
|
}}
|
||||||
if registry.typedEntries == nil {
|
if registry.typedEntries == nil {
|
||||||
registry.typedEntries = map[string]typedExtractorEntry{}
|
registry.typedEntries = map[string]typedExtractorEntry{}
|
||||||
|
|||||||
@@ -41,6 +41,54 @@ func operationReferenceSet(input RunInput, target ResolvedReferenceTarget) contr
|
|||||||
return CloneReferenceSet(target.ReferenceSet)
|
return CloneReferenceSet(target.ReferenceSet)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// referenceTargetReuseEligible reports whether every generated artifact in a
|
||||||
|
// stage's reference set descends exclusively from fully validated work. Static
|
||||||
|
// references and callers that do not supply lineage metadata are reusable.
|
||||||
|
func referenceTargetReuseEligible(input RunInput, target ResolvedReferenceTarget) bool {
|
||||||
|
if input.referenceReuseEligibility == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
eligible, ok := input.referenceReuseEligibility[keyForReferenceTarget(target)]
|
||||||
|
return !ok || eligible
|
||||||
|
}
|
||||||
|
|
||||||
|
func laneReferencesReuseEligible(input RunInput, lane ResolvedArtifactLane) bool {
|
||||||
|
return referenceTargetReuseEligible(input, lane.ExtractReferences) &&
|
||||||
|
referenceTargetReuseEligible(input, lane.MergeReferences) &&
|
||||||
|
referenceTargetReuseEligible(input, lane.NormalizeReferences)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildStepReferenceReuseEligibility carries validation completeness alongside
|
||||||
|
// generated references without exposing the internal lineage flag in artifact
|
||||||
|
// payloads. A target becomes ineligible when any generated input is ineligible.
|
||||||
|
func buildStepReferenceReuseEligibility(step PreparedPipelineStep, outputs map[generatedOutputKey]bool) map[referenceTargetKey]bool {
|
||||||
|
eligibility := make(map[referenceTargetKey]bool)
|
||||||
|
for _, prepared := range step.lanes {
|
||||||
|
lane := prepared.resolved
|
||||||
|
for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||||
|
generated := false
|
||||||
|
eligible := true
|
||||||
|
for _, binding := range target.Bindings {
|
||||||
|
if binding.Artifact == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
generated = true
|
||||||
|
producer := generatedOutputKeyFor(binding.Artifact.Step, binding.Artifact.Lane)
|
||||||
|
if reusable, ok := outputs[producer]; ok && !reusable {
|
||||||
|
eligible = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if generated {
|
||||||
|
eligibility[keyForReferenceTarget(target)] = eligible
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(eligibility) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return eligibility
|
||||||
|
}
|
||||||
|
|
||||||
// buildStepReferenceSets resolves every generated binding for a step before
|
// buildStepReferenceSets resolves every generated binding for a step before
|
||||||
// any lane in that step is allowed to start. Each returned set is a fresh
|
// any lane in that step is allowed to start. Each returned set is a fresh
|
||||||
// operation-time view; prepared reference sets are never modified.
|
// operation-time view; prepared reference sets are never modified.
|
||||||
|
|||||||
@@ -85,11 +85,19 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
|||||||
}
|
}
|
||||||
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
||||||
}
|
}
|
||||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone merge correction: %w", err)
|
||||||
|
}
|
||||||
|
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
|
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone merge model candidate: %w", err)
|
||||||
|
}
|
||||||
|
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), ModelCandidate: candidate}, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ type ModuleSpec struct {
|
|||||||
Key string
|
Key string
|
||||||
Stage ModuleStage
|
Stage ModuleStage
|
||||||
ExecutionClass contracts.ExecutionClass
|
ExecutionClass contracts.ExecutionClass
|
||||||
|
CorrectionProtocol contracts.CorrectionProtocol
|
||||||
ArtifactKind contracts.ArtifactKind
|
ArtifactKind contracts.ArtifactKind
|
||||||
Provides []string
|
Provides []string
|
||||||
Requires []string
|
Requires []string
|
||||||
@@ -32,10 +33,12 @@ type ModuleSpec struct {
|
|||||||
|
|
||||||
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||||
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
|
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
|
||||||
|
correctionProtocol := contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol)))
|
||||||
return ModuleSpec{
|
return ModuleSpec{
|
||||||
Key: strings.TrimSpace(spec.Key),
|
Key: strings.TrimSpace(spec.Key),
|
||||||
Stage: spec.Stage,
|
Stage: spec.Stage,
|
||||||
ExecutionClass: executionClass,
|
ExecutionClass: executionClass,
|
||||||
|
CorrectionProtocol: correctionProtocol,
|
||||||
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
||||||
Provides: normalizeCapabilities(spec.Provides),
|
Provides: normalizeCapabilities(spec.Provides),
|
||||||
Requires: normalizeCapabilities(spec.Requires),
|
Requires: normalizeCapabilities(spec.Requires),
|
||||||
@@ -73,6 +76,7 @@ func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
|
|||||||
Key: spec.Key,
|
Key: spec.Key,
|
||||||
Stage: spec.Stage,
|
Stage: spec.Stage,
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
CorrectionProtocol: spec.CorrectionProtocol,
|
||||||
ArtifactKind: spec.ArtifactKind,
|
ArtifactKind: spec.ArtifactKind,
|
||||||
Provides: append([]string(nil), spec.Provides...),
|
Provides: append([]string(nil), spec.Provides...),
|
||||||
Requires: append([]string(nil), spec.Requires...),
|
Requires: append([]string(nil), spec.Requires...),
|
||||||
@@ -93,6 +97,19 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
|||||||
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
|
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
|
||||||
}
|
}
|
||||||
|
if spec.CorrectionProtocol != "" {
|
||||||
|
if err := spec.CorrectionProtocol.Validate(); err != nil {
|
||||||
|
return fmt.Errorf("%s %q correction protocol: %w", kind, spec.Key, err)
|
||||||
|
}
|
||||||
|
if spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
return fmt.Errorf("%s %q correction protocol requires an LLM-backed execution class", kind, spec.Key)
|
||||||
|
}
|
||||||
|
switch spec.Stage {
|
||||||
|
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s %q correction protocol is not supported for %q stage", kind, spec.Key, spec.Stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
|
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
|
||||||
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
|
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,14 +32,70 @@ func TestValidateModuleSpecRequiresSupportedExecutionClass(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneModuleSpecPreservesExecutionClass(t *testing.T) {
|
func TestCloneModuleSpecPreservesCorrectionProtocol(t *testing.T) {
|
||||||
spec := normalizeModuleSpec(ModuleSpec{Key: " module ", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked})
|
spec := normalizeModuleSpec(ModuleSpec{
|
||||||
|
Key: " module ",
|
||||||
|
Stage: StageChunk,
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
CorrectionProtocol: " single_response_v1 ",
|
||||||
|
})
|
||||||
|
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||||
|
t.Fatalf("normalized CorrectionProtocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
}
|
||||||
cloned := cloneModuleSpec(spec)
|
cloned := cloneModuleSpec(spec)
|
||||||
if !reflect.DeepEqual(cloned, spec) {
|
if !reflect.DeepEqual(cloned, spec) {
|
||||||
t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec)
|
t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateModuleSpecCorrectionProtocolEligibility(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
stage ModuleStage
|
||||||
|
class contracts.ExecutionClass
|
||||||
|
protocol contracts.CorrectionProtocol
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "LLM chunk", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||||
|
{name: "LLM extract", stage: StageExtract, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||||
|
{name: "LLM merge", stage: StageMerge, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||||
|
{name: "LLM normalize", stage: StageNormalize, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||||
|
{name: "deterministic chunk", stage: StageChunk, class: contracts.ExecutionClassDeterministic, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "LLM-backed"},
|
||||||
|
{name: "input", stage: StageInput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
||||||
|
{name: "output", stage: StageOutput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
||||||
|
{name: "unknown protocol", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: "unsupported", want: "unsupported correction protocol"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
spec := normalizeModuleSpec(ModuleSpec{
|
||||||
|
Key: "module",
|
||||||
|
Stage: test.stage,
|
||||||
|
ExecutionClass: test.class,
|
||||||
|
CorrectionProtocol: test.protocol,
|
||||||
|
})
|
||||||
|
err := validateModuleSpec("module", test.stage, spec)
|
||||||
|
if test.want == "" && err != nil {
|
||||||
|
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
|
||||||
|
t.Fatalf("validateModuleSpec() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeValidatorSpecRejectsCorrectionProtocol(t *testing.T) {
|
||||||
|
_, err := normalizeValidatorSpec(ValidatorSpec{
|
||||||
|
Key: "validator",
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "correction protocol") {
|
||||||
|
t.Fatalf("normalizeValidatorSpec() error = %v, want correction protocol error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
|
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -76,11 +76,19 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone normalize correction: %w", err)
|
||||||
|
}
|
||||||
|
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), Retry: cloneNormalizeRetry(result.Retry)}, nil
|
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return erasedTypedResult{}, fmt.Errorf("clone normalize model candidate: %w", err)
|
||||||
|
}
|
||||||
|
return erasedTypedResult{Value: result.Value, Warnings: cloneWarnings(result.Warnings), Retry: cloneNormalizeRetry(result.Retry), ModelCandidate: candidate}, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
type PreparedPipeline struct {
|
type PreparedPipeline struct {
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
|
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
||||||
Steps []PreparedPipelineStep
|
Steps []PreparedPipelineStep
|
||||||
Output ModuleBinding
|
Output ModuleBinding
|
||||||
|
|
||||||
@@ -37,6 +38,9 @@ type PreparedPipelineStep struct {
|
|||||||
|
|
||||||
type PreparedArtifactLane struct {
|
type PreparedArtifactLane struct {
|
||||||
Resolved ResolvedArtifactLane
|
Resolved ResolvedArtifactLane
|
||||||
|
ExtractCorrectionProtocol contracts.CorrectionProtocol
|
||||||
|
MergeCorrectionProtocol contracts.CorrectionProtocol
|
||||||
|
NormalizeCorrectionProtocol contracts.CorrectionProtocol
|
||||||
}
|
}
|
||||||
|
|
||||||
type preparedLaneExecutor struct {
|
type preparedLaneExecutor struct {
|
||||||
@@ -79,6 +83,7 @@ type preparedValidator struct {
|
|||||||
typedValidate typedValidateOperation
|
typedValidate typedValidateOperation
|
||||||
chunk contracts.ChunkValidator
|
chunk contracts.ChunkValidator
|
||||||
serialized contracts.SerializedValidator
|
serialized contracts.SerializedValidator
|
||||||
|
position int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare validates all configured options and constructs every selected
|
// Prepare validates all configured options and constructs every selected
|
||||||
@@ -87,6 +92,9 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
if err := validateResolvedPipeline(resolved); err != nil {
|
if err := validateResolvedPipeline(resolved); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := validateCorrectionRetryCapabilities(resolved); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if err := validateRegistrySet(resolved, registries); err != nil {
|
if err := validateRegistrySet(resolved, registries); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -94,6 +102,7 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
prepared := &PreparedPipeline{
|
prepared := &PreparedPipeline{
|
||||||
Input: cloneModuleBinding(stable.Input),
|
Input: cloneModuleBinding(stable.Input),
|
||||||
Chunk: cloneModuleBinding(stable.Chunk),
|
Chunk: cloneModuleBinding(stable.Chunk),
|
||||||
|
ChunkCorrectionProtocol: stable.ChunkCorrectionProtocol,
|
||||||
Output: cloneModuleBinding(stable.Output),
|
Output: cloneModuleBinding(stable.Output),
|
||||||
resolved: stable,
|
resolved: stable,
|
||||||
dependencies: deps,
|
dependencies: deps,
|
||||||
@@ -131,7 +140,12 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{Resolved: cloneResolvedArtifactLane(lane)})
|
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{
|
||||||
|
Resolved: cloneResolvedArtifactLane(lane),
|
||||||
|
ExtractCorrectionProtocol: lane.ExtractCorrectionProtocol,
|
||||||
|
MergeCorrectionProtocol: lane.MergeCorrectionProtocol,
|
||||||
|
NormalizeCorrectionProtocol: lane.NormalizeCorrectionProtocol,
|
||||||
|
})
|
||||||
preparedStep.lanes = append(preparedStep.lanes, executor)
|
preparedStep.lanes = append(preparedStep.lanes, executor)
|
||||||
}
|
}
|
||||||
prepared.Steps[stepIndex] = preparedStep
|
prepared.Steps[stepIndex] = preparedStep
|
||||||
@@ -199,6 +213,40 @@ func prepareEvidencePlan(resolved ResolvedPipeline, registries Registries, outpu
|
|||||||
return plan, nil
|
return plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCorrectionRetryCapabilities(pipeline ResolvedPipeline) error {
|
||||||
|
validate := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass, protocol contracts.CorrectionProtocol) error {
|
||||||
|
if executionClass != contracts.ExecutionClassLLMBacked || binding.Retries == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
chain := resolvedValidatorChain(stage, laneID, binding.Module, pipeline.ValidatorChains)
|
||||||
|
if len(chain.Validators) == 0 || protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if laneID == "" {
|
||||||
|
return fmt.Errorf("pipeline %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, laneID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := validate(StageChunk, "", pipeline.Chunk, pipeline.ChunkExecutionClass, pipeline.ChunkCorrectionProtocol); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, step := range pipeline.Steps {
|
||||||
|
for _, lane := range step.ArtifactLanes {
|
||||||
|
if err := validate(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass, lane.ExtractCorrectionProtocol); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validate(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass, lane.MergeCorrectionProtocol); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validate(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizeCorrectionProtocol); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
|
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
|
||||||
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
|
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
|
||||||
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
||||||
@@ -406,6 +454,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
|
|||||||
|
|
||||||
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
||||||
out := in
|
out := in
|
||||||
|
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
|
|||||||
391
internal/framework/pipeline/producer_attempts.go
Normal file
391
internal/framework/pipeline/producer_attempts.go
Normal file
@@ -0,0 +1,391 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// producerAttemptKind records why a producer invocation followed the prior
|
||||||
|
// one. It is deliberately independent of any artifact family.
|
||||||
|
type producerAttemptKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
producerAttemptInitial producerAttemptKind = "initial"
|
||||||
|
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
|
||||||
|
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
|
||||||
|
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
|
||||||
|
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
|
||||||
|
)
|
||||||
|
|
||||||
|
type producerAttemptOutcome string
|
||||||
|
|
||||||
|
const (
|
||||||
|
producerAttemptAccepted producerAttemptOutcome = "accepted"
|
||||||
|
producerAttemptRejected producerAttemptOutcome = "rejected"
|
||||||
|
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
|
||||||
|
producerAttemptRetried producerAttemptOutcome = "retried"
|
||||||
|
producerAttemptFailed producerAttemptOutcome = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
type producerTerminalAction string
|
||||||
|
|
||||||
|
const (
|
||||||
|
producerTerminalAccepted producerTerminalAction = "accepted"
|
||||||
|
producerTerminalRejected producerTerminalAction = "reject_output"
|
||||||
|
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
|
||||||
|
producerTerminalFailed producerTerminalAction = "fail_run"
|
||||||
|
)
|
||||||
|
|
||||||
|
// producerAttemptRequest contains only attempt-local control material. The
|
||||||
|
// producer reconstructs its ordinary request from its own durable inputs.
|
||||||
|
type producerAttemptRequest struct {
|
||||||
|
Number int
|
||||||
|
Kind producerAttemptKind
|
||||||
|
Correction *contracts.SemanticCorrection
|
||||||
|
}
|
||||||
|
|
||||||
|
// producerRetryDirective asks for another producer invocation while retaining
|
||||||
|
// the current value as a safe fallback if its shared budget is exhausted.
|
||||||
|
// Artifact-specific adapters are responsible for validating and populating it.
|
||||||
|
type producerRetryDirective struct {
|
||||||
|
FallbackWarnings []contracts.Warning
|
||||||
|
}
|
||||||
|
|
||||||
|
func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
||||||
|
if directive == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &producerRetryDirective{FallbackWarnings: cloneWarnings(directive.FallbackWarnings)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
||||||
|
// opaque to the state machine; Candidate is the attempt-local response that
|
||||||
|
// may support semantic correction.
|
||||||
|
type producerAttemptOutput struct {
|
||||||
|
Value any
|
||||||
|
Candidate *contracts.ModelCandidate
|
||||||
|
Warnings []contracts.Warning
|
||||||
|
Retry *producerRetryDirective
|
||||||
|
}
|
||||||
|
|
||||||
|
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
|
||||||
|
candidate, err := contracts.CloneModelCandidate(output.Candidate)
|
||||||
|
if err != nil {
|
||||||
|
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
|
||||||
|
}
|
||||||
|
return producerAttemptOutput{
|
||||||
|
Value: output.Value,
|
||||||
|
Candidate: candidate,
|
||||||
|
Warnings: cloneWarnings(output.Warnings),
|
||||||
|
Retry: output.Retry.clone(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
|
||||||
|
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
|
||||||
|
|
||||||
|
type producerAttemptConfig struct {
|
||||||
|
Retries int
|
||||||
|
Policy ValidationPolicy
|
||||||
|
AllowStructuralRetry bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// producerAttemptProvenance is ordered by producer invocation. It retains the
|
||||||
|
// settled validation report for later debug and durable-provenance adapters.
|
||||||
|
type producerAttemptProvenance struct {
|
||||||
|
Number int
|
||||||
|
Kind producerAttemptKind
|
||||||
|
Outcome producerAttemptOutcome
|
||||||
|
Validation validationReport
|
||||||
|
}
|
||||||
|
|
||||||
|
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
|
||||||
|
provenance.Validation = cloneValidationReport(provenance.Validation)
|
||||||
|
return provenance
|
||||||
|
}
|
||||||
|
|
||||||
|
// producerAttemptTerminal describes the state-machine decision without
|
||||||
|
// materializing an artifact or persisting stage-specific diagnostics.
|
||||||
|
type producerAttemptTerminal struct {
|
||||||
|
Action producerTerminalAction
|
||||||
|
Value any
|
||||||
|
Warnings []contracts.Warning
|
||||||
|
Rejection *contracts.RejectedOutput
|
||||||
|
Validation validationReport
|
||||||
|
ValidationIncomplete bool
|
||||||
|
Provenance []producerAttemptProvenance
|
||||||
|
}
|
||||||
|
|
||||||
|
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
|
||||||
|
terminal.Warnings = cloneWarnings(terminal.Warnings)
|
||||||
|
if terminal.Rejection != nil {
|
||||||
|
rejection := *terminal.Rejection
|
||||||
|
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
|
||||||
|
terminal.Rejection = &rejection
|
||||||
|
}
|
||||||
|
terminal.Validation = cloneValidationReport(terminal.Validation)
|
||||||
|
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
|
||||||
|
return terminal
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
|
||||||
|
if len(provenance) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make([]producerAttemptProvenance, len(provenance))
|
||||||
|
for index, item := range provenance {
|
||||||
|
cloned[index] = item.clone()
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidationReport(report validationReport) validationReport {
|
||||||
|
return validationReport{records: report.Records()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
|
||||||
|
}
|
||||||
|
if config.Retries < 0 {
|
||||||
|
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
|
||||||
|
}
|
||||||
|
if produce == nil {
|
||||||
|
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
|
||||||
|
}
|
||||||
|
if validate == nil {
|
||||||
|
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
attemptLimit := config.Retries + 1
|
||||||
|
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
|
||||||
|
kind := producerAttemptInitial
|
||||||
|
var correction *contracts.SemanticCorrection
|
||||||
|
|
||||||
|
for number := 1; number <= attemptLimit; number++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return failedProducerAttempt(provenance), err
|
||||||
|
}
|
||||||
|
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
|
||||||
|
if err != nil {
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
|
||||||
|
}
|
||||||
|
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
|
||||||
|
if err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||||
|
if isImmediateProducerFailure(err) {
|
||||||
|
return failedProducerAttempt(provenance), err
|
||||||
|
}
|
||||||
|
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
|
||||||
|
if number < attemptLimit {
|
||||||
|
kind, correction = producerAttemptStructuralRetry, nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
|
||||||
|
}
|
||||||
|
if number < attemptLimit {
|
||||||
|
kind, correction = producerAttemptOperationalRetry, nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err = output.clone()
|
||||||
|
if err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||||
|
return failedProducerAttempt(provenance), err
|
||||||
|
}
|
||||||
|
if output.Retry != nil && number < attemptLimit {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
||||||
|
kind, correction = producerAttemptModuleRetry, nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if output.Retry != nil {
|
||||||
|
output.Warnings = append(output.Warnings, cloneWarnings(output.Retry.FallbackWarnings)...)
|
||||||
|
}
|
||||||
|
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
|
||||||
|
if err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := validate(ctx, output)
|
||||||
|
if err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||||
|
return failedProducerAttempt(provenance), err
|
||||||
|
}
|
||||||
|
report = cloneValidationReport(report)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||||
|
return failedProducerAttempt(provenance), err
|
||||||
|
}
|
||||||
|
|
||||||
|
if rejection := report.FirstRejection(); rejection != nil {
|
||||||
|
if correctionCandidate != nil {
|
||||||
|
if err := correctionCandidate.Validate(); err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
||||||
|
correctionRequest, guidanceErr := report.CorrectionRequest()
|
||||||
|
if guidanceErr != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
|
||||||
|
}
|
||||||
|
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
|
||||||
|
if err != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
|
||||||
|
}
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
|
||||||
|
kind = producerAttemptSemanticRetry
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
|
||||||
|
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
|
||||||
|
}
|
||||||
|
|
||||||
|
if incomplete := firstIncompleteValidation(report); incomplete != nil {
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
|
||||||
|
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
|
||||||
|
warnings := terminalWarnings(output, report)
|
||||||
|
warnings = append(warnings, incompleteValidationWarnings(report)...)
|
||||||
|
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Warnings: warnings, Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||||
|
}
|
||||||
|
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
|
||||||
|
}
|
||||||
|
|
||||||
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
|
||||||
|
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Warnings: terminalWarnings(output, report), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isImmediateProducerFailure(err error) bool {
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var debugErr *attemptDebugPersistenceError
|
||||||
|
return errors.As(err, &debugErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
|
||||||
|
switch policy.ProducerStructuralFailure {
|
||||||
|
case ProducerStructuralFailureRejectOutput:
|
||||||
|
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||||
|
case ProducerStructuralFailureFailRun:
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
|
||||||
|
default:
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
|
||||||
|
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
|
||||||
|
switch policy.SemanticRejection {
|
||||||
|
case SemanticRejectionRejectOutput:
|
||||||
|
return producerAttemptTerminal{Action: producerTerminalRejected, Warnings: terminalWarnings(output, report), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||||
|
case SemanticRejectionFailRun:
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
|
||||||
|
default:
|
||||||
|
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstIncompleteValidation(report validationReport) *validationRecord {
|
||||||
|
for _, record := range report.records {
|
||||||
|
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
||||||
|
clone := record.clone()
|
||||||
|
return &clone
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func terminalWarnings(output producerAttemptOutput, report validationReport) []contracts.Warning {
|
||||||
|
warnings := cloneWarnings(output.Warnings)
|
||||||
|
warnings = append(warnings, report.Warnings()...)
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
// incompleteValidationWarnings reports only validators that exhausted their
|
||||||
|
// execution budget. It never reports rejected candidates, and it uses fixed
|
||||||
|
// text so provider errors and correction content cannot cross this boundary.
|
||||||
|
func incompleteValidationWarnings(report validationReport) []contracts.Warning {
|
||||||
|
warnings := make([]contracts.Warning, 0)
|
||||||
|
for _, record := range report.records {
|
||||||
|
if record.outcome != validationFailed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: record.validatorName,
|
||||||
|
ReasonCode: "validator_execution_incomplete",
|
||||||
|
Message: "Validator execution did not complete within its configured budget.",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
// validationSummary projects a terminal state-machine result into the durable
|
||||||
|
// bounded representation used by manifests, rejections, and receipts.
|
||||||
|
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
|
||||||
|
summary := artifacts.ValidationSummary{
|
||||||
|
Stage: string(stage),
|
||||||
|
StepID: stepID,
|
||||||
|
LaneID: laneID,
|
||||||
|
ModuleKey: moduleKey,
|
||||||
|
ChunkID: chunkID,
|
||||||
|
ChunkIndex: chunkIndex,
|
||||||
|
ProducerAttemptCount: len(terminal.Provenance),
|
||||||
|
TerminalAction: string(terminal.Action),
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case terminal.Action == producerTerminalRejected:
|
||||||
|
summary.Status = "rejected"
|
||||||
|
case terminal.Action == producerTerminalFailed:
|
||||||
|
summary.Status = "incomplete"
|
||||||
|
case terminal.ValidationIncomplete:
|
||||||
|
summary.Status = "incomplete"
|
||||||
|
default:
|
||||||
|
summary.Status = "complete"
|
||||||
|
}
|
||||||
|
seenValidators := make(map[string]struct{})
|
||||||
|
seenReasons := make(map[string]struct{})
|
||||||
|
seenIncomplete := make(map[string]struct{})
|
||||||
|
for _, record := range terminal.Validation.records {
|
||||||
|
if record.reasonCode != "" {
|
||||||
|
if _, exists := seenReasons[record.reasonCode]; !exists {
|
||||||
|
seenReasons[record.reasonCode] = struct{}{}
|
||||||
|
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if record.outcome == validationRejected {
|
||||||
|
if _, exists := seenValidators[record.validatorName]; !exists {
|
||||||
|
seenValidators[record.validatorName] = struct{}{}
|
||||||
|
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
||||||
|
if _, exists := seenIncomplete[record.validatorName]; !exists {
|
||||||
|
seenIncomplete[record.validatorName] = struct{}{}
|
||||||
|
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
|
||||||
|
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
|
||||||
|
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
}
|
||||||
|
|
||||||
|
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
|
||||||
|
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
|
||||||
|
}
|
||||||
414
internal/framework/pipeline/producer_attempts_test.go
Normal file
414
internal/framework/pipeline/producer_attempts_test.go
Normal file
@@ -0,0 +1,414 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) {
|
||||||
|
candidate := attemptCandidate(t, "first response")
|
||||||
|
producerCalls := 0
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
producerCalls++
|
||||||
|
switch request.Number {
|
||||||
|
case 1:
|
||||||
|
return producerAttemptOutput{}, errors.New("temporary producer failure")
|
||||||
|
case 2:
|
||||||
|
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||||
|
case 3:
|
||||||
|
return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||||
|
return producerAttemptOutput{}, nil
|
||||||
|
}
|
||||||
|
}, approveAttempt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||||
|
t.Fatalf("terminal = %#v, want accepted value", terminal)
|
||||||
|
}
|
||||||
|
if producerCalls != 3 {
|
||||||
|
t.Fatalf("producer calls = %d, want 3", producerCalls)
|
||||||
|
}
|
||||||
|
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) {
|
||||||
|
t.Fatalf("attempt kinds = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) {
|
||||||
|
first := attemptCandidate(t, "first response")
|
||||||
|
second := attemptCandidate(t, "second response")
|
||||||
|
var corrections []*contracts.SemanticCorrection
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
if request.Correction != nil {
|
||||||
|
corrections = append(corrections, request.Correction)
|
||||||
|
}
|
||||||
|
switch request.Number {
|
||||||
|
case 1:
|
||||||
|
return producerAttemptOutput{Value: "one", Candidate: first}, nil
|
||||||
|
case 2:
|
||||||
|
return producerAttemptOutput{Value: "two", Candidate: second}, nil
|
||||||
|
case 3:
|
||||||
|
return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||||
|
return producerAttemptOutput{}, nil
|
||||||
|
}
|
||||||
|
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||||
|
if output.Value == "three" {
|
||||||
|
return validationReport{}, nil
|
||||||
|
}
|
||||||
|
return rejectedAttemptReport("semantic_defect"), nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalAccepted {
|
||||||
|
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
||||||
|
}
|
||||||
|
if len(corrections) != 2 {
|
||||||
|
t.Fatalf("correction count = %d, want 2", len(corrections))
|
||||||
|
}
|
||||||
|
if corrections[0] == corrections[1] {
|
||||||
|
t.Fatal("semantic corrections reused the same pointer")
|
||||||
|
}
|
||||||
|
if got := string(corrections[0].AssistantResponse); got != "first response" {
|
||||||
|
t.Fatalf("first correction response = %q", got)
|
||||||
|
}
|
||||||
|
if got := string(corrections[1].AssistantResponse); got != "second response" {
|
||||||
|
t.Fatalf("second correction response = %q", got)
|
||||||
|
}
|
||||||
|
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) {
|
||||||
|
t.Fatalf("attempt kinds = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
config producerAttemptConfig
|
||||||
|
produce producerAttemptProducer
|
||||||
|
validate producerAttemptValidator
|
||||||
|
wantAction producerTerminalAction
|
||||||
|
wantError bool
|
||||||
|
wantValue any
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "structural reject output",
|
||||||
|
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||||
|
},
|
||||||
|
validate: approveAttempt,
|
||||||
|
wantAction: producerTerminalRejected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "structural fail run",
|
||||||
|
config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||||
|
},
|
||||||
|
validate: approveAttempt,
|
||||||
|
wantAction: producerTerminalFailed,
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "operational failure after retry exhaustion",
|
||||||
|
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, errors.New("producer unavailable")
|
||||||
|
},
|
||||||
|
validate: approveAttempt,
|
||||||
|
wantAction: producerTerminalFailed,
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "semantic reject output",
|
||||||
|
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil
|
||||||
|
},
|
||||||
|
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return rejectedAttemptReport("bad"), nil
|
||||||
|
},
|
||||||
|
wantAction: producerTerminalRejected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "semantic fail run",
|
||||||
|
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil
|
||||||
|
},
|
||||||
|
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return rejectedAttemptReport("bad"), nil
|
||||||
|
},
|
||||||
|
wantAction: producerTerminalFailed,
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validator failure warns and continues",
|
||||||
|
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{Value: "kept"}, nil
|
||||||
|
},
|
||||||
|
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return failedAttemptReport(), nil
|
||||||
|
},
|
||||||
|
wantAction: producerTerminalIncompleteAccepted,
|
||||||
|
wantValue: "kept",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validator failure fails run",
|
||||||
|
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}},
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, nil
|
||||||
|
},
|
||||||
|
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return failedAttemptReport(), nil
|
||||||
|
},
|
||||||
|
wantAction: producerTerminalFailed,
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate)
|
||||||
|
if (err != nil) != test.wantError {
|
||||||
|
t.Fatalf("error = %v, want error %t", err, test.wantError)
|
||||||
|
}
|
||||||
|
if terminal.Action != test.wantAction {
|
||||||
|
t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction)
|
||||||
|
}
|
||||||
|
if terminal.Value != test.wantValue {
|
||||||
|
t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue)
|
||||||
|
}
|
||||||
|
if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) {
|
||||||
|
t.Fatalf("rejected terminal = %#v, want rejection without value", terminal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
policy := DefaultValidationPolicy()
|
||||||
|
policy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
calls++
|
||||||
|
return producerAttemptOutput{Value: "deterministic"}, nil
|
||||||
|
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return rejectedAttemptReport("deterministic_rejection"), nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 {
|
||||||
|
t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||||
|
t.Run("retry", func(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
calls++
|
||||||
|
if request.Number == 1 {
|
||||||
|
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
||||||
|
}
|
||||||
|
return producerAttemptOutput{Value: "replacement"}, nil
|
||||||
|
}, approveAttempt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 {
|
||||||
|
t.Fatalf("terminal = %#v, calls = %d", terminal, calls)
|
||||||
|
}
|
||||||
|
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||||
|
t.Fatalf("attempt kinds = %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("fallback", func(t *testing.T) {
|
||||||
|
fallbackWarning := contracts.Warning{ReasonCode: "fallback", Message: "fallback warning"}
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackWarnings: []contracts.Warning{fallbackWarning}}}, nil
|
||||||
|
}, approveAttempt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Warnings, []contracts.Warning{fallbackWarning}) {
|
||||||
|
t.Fatalf("terminal = %#v", terminal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) {
|
||||||
|
firstWarnings := []contracts.Warning{{ReasonCode: "discarded", Message: "discarded warning"}}
|
||||||
|
secondWarnings := []contracts.Warning{{ReasonCode: "accepted", Message: "accepted warning"}}
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
if request.Number == 1 {
|
||||||
|
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Warnings: firstWarnings}, nil
|
||||||
|
}
|
||||||
|
return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Warnings: secondWarnings}, nil
|
||||||
|
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||||
|
if output.Value == "first" {
|
||||||
|
report := rejectedAttemptReport("defect")
|
||||||
|
report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")})
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
return validationReport{}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalAccepted {
|
||||||
|
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
||||||
|
}
|
||||||
|
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) {
|
||||||
|
t.Fatalf("attempt kinds = %v", got)
|
||||||
|
}
|
||||||
|
if got := terminal.Warnings; !reflect.DeepEqual(got, secondWarnings) {
|
||||||
|
t.Fatalf("terminal warnings = %#v, want %#v", got, secondWarnings)
|
||||||
|
}
|
||||||
|
if len(terminal.Provenance[0].Validation.records) != 2 {
|
||||||
|
t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) {
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
if request.Number == 1 {
|
||||||
|
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil
|
||||||
|
}
|
||||||
|
return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil
|
||||||
|
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||||
|
if output.Value == "first" {
|
||||||
|
return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil
|
||||||
|
}
|
||||||
|
return validationReport{}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3)
|
||||||
|
if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 {
|
||||||
|
t.Fatalf("corrected success summary retained prior findings: %#v", summary)
|
||||||
|
}
|
||||||
|
if len(terminal.Warnings) != 0 {
|
||||||
|
t.Fatalf("corrected success warnings = %#v, want none", terminal.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) {
|
||||||
|
report := validationReport{records: []validationRecord{
|
||||||
|
{validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")},
|
||||||
|
{validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"},
|
||||||
|
{validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")},
|
||||||
|
}}
|
||||||
|
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{Value: "candidate"}, nil
|
||||||
|
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return report, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||||
|
}
|
||||||
|
if terminal.Action != producerTerminalIncompleteAccepted {
|
||||||
|
t.Fatalf("terminal action = %q", terminal.Action)
|
||||||
|
}
|
||||||
|
if got, want := terminal.Warnings, []contracts.Warning{
|
||||||
|
{Scope: "first", ReasonCode: "validator_execution_incomplete", Message: "Validator execution did not complete within its configured budget."},
|
||||||
|
{Scope: "third", ReasonCode: "validator_execution_incomplete", Message: "Validator execution did not complete within its configured budget."},
|
||||||
|
}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("warnings = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0)
|
||||||
|
if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
context context.Context
|
||||||
|
produce producerAttemptProducer
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "cancelled context",
|
||||||
|
context: cancelledAttemptContext(),
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "debug persistence failure",
|
||||||
|
context: context.Background(),
|
||||||
|
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||||
|
calls++
|
||||||
|
return test.produce(ctx, request)
|
||||||
|
}
|
||||||
|
terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt)
|
||||||
|
if err == nil || terminal.Action != producerTerminalFailed {
|
||||||
|
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||||
|
}
|
||||||
|
if test.name == "cancelled context" && calls != 0 {
|
||||||
|
t.Fatalf("cancelled producer calls = %d, want 0", calls)
|
||||||
|
}
|
||||||
|
if test.name == "debug persistence failure" && calls != 1 {
|
||||||
|
t.Fatalf("debug failure producer calls = %d, want 1", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate {
|
||||||
|
t.Helper()
|
||||||
|
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewModelCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||||
|
return validationReport{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectedAttemptReport(reason string) validationReport {
|
||||||
|
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func failedAttemptReport() validationReport {
|
||||||
|
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind {
|
||||||
|
kinds := make([]producerAttemptKind, len(provenance))
|
||||||
|
for index, item := range provenance {
|
||||||
|
kinds[index] = item.Kind
|
||||||
|
}
|
||||||
|
return kinds
|
||||||
|
}
|
||||||
|
|
||||||
|
func cancelledAttemptContext() context.Context {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
@@ -23,6 +23,8 @@ const (
|
|||||||
type ModuleBinding struct {
|
type ModuleBinding struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||||
Retries int `json:"retries,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
@@ -82,6 +84,8 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
|||||||
type moduleBindingJSON struct {
|
type moduleBindingJSON struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||||
Retries int `json:"retries,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
@@ -90,6 +94,8 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
|||||||
out := moduleBindingJSON{
|
out := moduleBindingJSON{
|
||||||
Module: binding.Module,
|
Module: binding.Module,
|
||||||
LLMProfile: binding.LLMProfile,
|
LLMProfile: binding.LLMProfile,
|
||||||
|
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
|
||||||
Retries: binding.Retries,
|
Retries: binding.Retries,
|
||||||
Options: binding.Options,
|
Options: binding.Options,
|
||||||
References: binding.References,
|
References: binding.References,
|
||||||
@@ -101,6 +107,44 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
|||||||
return json.Marshal(out)
|
return json.Marshal(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||||
|
if binding == nil {
|
||||||
|
return fmt.Errorf("module binding must not be nil")
|
||||||
|
}
|
||||||
|
type moduleBindingJSON struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||||
|
Retries int `json:"retries,omitempty"`
|
||||||
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
|
Validators json.RawMessage `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
var decoded moduleBindingJSON
|
||||||
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out := ModuleBinding{
|
||||||
|
Module: decoded.Module,
|
||||||
|
LLMProfile: decoded.LLMProfile,
|
||||||
|
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
|
||||||
|
Retries: decoded.Retries,
|
||||||
|
Options: decoded.Options,
|
||||||
|
References: decoded.References,
|
||||||
|
}
|
||||||
|
if len(decoded.Validators) > 0 && string(decoded.Validators) != "null" {
|
||||||
|
var validators []ModuleBinding
|
||||||
|
if err := json.Unmarshal(decoded.Validators, &validators); err != nil {
|
||||||
|
return fmt.Errorf("decode module binding validators: %w", err)
|
||||||
|
}
|
||||||
|
out.Validators = ValidatorOverride{Set: true, Validators: validators}
|
||||||
|
}
|
||||||
|
*binding = cloneModuleBinding(out)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
Extract ModuleBinding `json:"extract"`
|
Extract ModuleBinding `json:"extract"`
|
||||||
Merge ModuleBinding `json:"merge,omitempty"`
|
Merge ModuleBinding `json:"merge,omitempty"`
|
||||||
@@ -118,6 +162,8 @@ type PipelineStepProfile struct {
|
|||||||
type PipelineProfile struct {
|
type PipelineProfile struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||||
Input ModuleBinding `json:"input"`
|
Input ModuleBinding `json:"input"`
|
||||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||||
@@ -167,10 +213,16 @@ type ResolvedArtifactLane struct {
|
|||||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||||
Extract ModuleBinding
|
Extract ModuleBinding
|
||||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||||
|
ExtractCorrectionProtocol contracts.CorrectionProtocol `json:"extract_correction_protocol,omitempty"`
|
||||||
|
ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
|
||||||
Merge ModuleBinding
|
Merge ModuleBinding
|
||||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||||
|
MergeCorrectionProtocol contracts.CorrectionProtocol `json:"merge_correction_protocol,omitempty"`
|
||||||
|
MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
|
||||||
Normalize ModuleBinding
|
Normalize ModuleBinding
|
||||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||||
|
NormalizeCorrectionProtocol contracts.CorrectionProtocol `json:"normalize_correction_protocol,omitempty"`
|
||||||
|
NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
|
||||||
Validators []ModuleBinding
|
Validators []ModuleBinding
|
||||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||||
@@ -199,10 +251,13 @@ type ResolvedValidator struct {
|
|||||||
type ResolvedPipeline struct {
|
type ResolvedPipeline struct {
|
||||||
ID string
|
ID string
|
||||||
Digest string
|
Digest string
|
||||||
|
ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||||
|
ChunkCorrectionProtocol contracts.CorrectionProtocol `json:"chunk_correction_protocol,omitempty"`
|
||||||
|
ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
|
||||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||||
Steps []ResolvedPipelineStep
|
Steps []ResolvedPipelineStep
|
||||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||||
@@ -375,10 +430,12 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
}
|
}
|
||||||
resolved := ResolvedPipeline{
|
resolved := ResolvedPipeline{
|
||||||
ID: pipelineID,
|
ID: pipelineID,
|
||||||
|
ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
|
||||||
Input: input,
|
Input: input,
|
||||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||||
Chunk: chunk,
|
Chunk: chunk,
|
||||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||||
|
ChunkCorrectionProtocol: chunkSpec.CorrectionProtocol,
|
||||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
Output: output,
|
Output: output,
|
||||||
}
|
}
|
||||||
@@ -450,6 +507,12 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
|
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
|
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
|
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
@@ -557,6 +620,7 @@ func resolveArtifactLane(
|
|||||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||||
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
|
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.ExtractExecutionClass = extractSpec.ExecutionClass
|
lane.ExtractExecutionClass = extractSpec.ExecutionClass
|
||||||
|
lane.ExtractCorrectionProtocol = extractSpec.CorrectionProtocol
|
||||||
capabilities.add(extractSpec.Provides...)
|
capabilities.add(extractSpec.Provides...)
|
||||||
|
|
||||||
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
|
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
|
||||||
@@ -583,6 +647,7 @@ func resolveArtifactLane(
|
|||||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||||
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
|
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.MergeExecutionClass = mergeSpec.ExecutionClass
|
lane.MergeExecutionClass = mergeSpec.ExecutionClass
|
||||||
|
lane.MergeCorrectionProtocol = mergeSpec.CorrectionProtocol
|
||||||
capabilities.add(mergeSpec.Provides...)
|
capabilities.add(mergeSpec.Provides...)
|
||||||
|
|
||||||
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
|
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
|
||||||
@@ -609,6 +674,7 @@ func resolveArtifactLane(
|
|||||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||||
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
|
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
|
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
|
||||||
|
lane.NormalizeCorrectionProtocol = normalizeSpec.CorrectionProtocol
|
||||||
capabilities.add(normalizeSpec.Provides...)
|
capabilities.add(normalizeSpec.Provides...)
|
||||||
|
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
@@ -870,6 +936,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q: %w", pipelineID, stage, chain.ModuleKey, err)
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q: %w", pipelineID, stage, chain.ModuleKey, err)
|
||||||
}
|
}
|
||||||
|
if validator.Retries > 0 && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator %q retries require an LLM-backed execution class", pipelineID, stage, validator.Module)
|
||||||
|
}
|
||||||
chain.Validators = append(chain.Validators, ResolvedValidator{
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
Binding: cloneModuleBinding(validator),
|
Binding: cloneModuleBinding(validator),
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
@@ -1284,6 +1353,8 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
|
|||||||
return ModuleBinding{
|
return ModuleBinding{
|
||||||
Module: module,
|
Module: module,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
|
||||||
Retries: binding.Retries,
|
Retries: binding.Retries,
|
||||||
Options: cloneOptions(binding.Options),
|
Options: cloneOptions(binding.Options),
|
||||||
References: references,
|
References: references,
|
||||||
@@ -1359,6 +1430,125 @@ func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, over
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pipelineAttempts *int) error {
|
||||||
|
apply := func(stage ModuleStage, laneID, module string, binding *ModuleBinding, executionClass contracts.ExecutionClass, kind string) error {
|
||||||
|
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||||
|
if executionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
if binding.StructuredOutputRepairAttempts != nil {
|
||||||
|
if laneID == "" {
|
||||||
|
return fmt.Errorf("pipeline %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, stage, module, kind, binding.Module)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, laneID, stage, module, kind, binding.Module)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if binding.StructuredOutputRepairAttempts == nil {
|
||||||
|
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(pipelineAttempts)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := apply(StageInput, "", resolved.Input.Module, &resolved.Input, resolved.InputExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageChunk, "", resolved.Chunk.Module, &resolved.Chunk, resolved.ChunkExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for stepIndex := range resolved.Steps {
|
||||||
|
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||||
|
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||||
|
if err := apply(StageExtract, lane.ID, lane.Extract.Module, &lane.Extract, lane.ExtractExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageMerge, lane.ID, lane.Merge.Module, &lane.Merge, lane.MergeExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageNormalize, lane.ID, lane.Normalize.Module, &lane.Normalize, lane.NormalizeExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := apply(StageOutput, "", resolved.Output.Module, &resolved.Output, resolved.OutputExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for chainIndex := range resolved.ValidatorChains {
|
||||||
|
chain := &resolved.ValidatorChains[chainIndex]
|
||||||
|
for validatorIndex := range chain.Validators {
|
||||||
|
validator := &chain.Validators[validatorIndex]
|
||||||
|
if err := apply(chain.Stage, chain.LaneID, chain.ModuleKey, &validator.Binding, validator.ExecutionClass, "validator"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
|
||||||
|
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
|
||||||
|
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
|
||||||
|
if binding.ValidationPolicy != nil {
|
||||||
|
if err := binding.ValidationPolicy.Validate(); err != nil {
|
||||||
|
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
|
||||||
|
}
|
||||||
|
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
|
||||||
|
if laneID == "" {
|
||||||
|
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
|
||||||
|
}
|
||||||
|
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if pipelinePolicy != nil {
|
||||||
|
if err := pipelinePolicy.Validate(); err != nil {
|
||||||
|
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if resolved.Input.ValidationPolicy != nil {
|
||||||
|
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
|
||||||
|
}
|
||||||
|
if resolved.Output.ValidationPolicy != nil {
|
||||||
|
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
|
||||||
|
}
|
||||||
|
for _, chain := range resolved.ValidatorChains {
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
if validator.Binding.ValidationPolicy != nil {
|
||||||
|
if chain.LaneID == "" {
|
||||||
|
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resolved.ChunkValidationPolicy = policy
|
||||||
|
for stepIndex := range resolved.Steps {
|
||||||
|
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||||
|
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||||
|
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lane.ExtractValidationPolicy = policy
|
||||||
|
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lane.MergeValidationPolicy = policy
|
||||||
|
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lane.NormalizeValidationPolicy = policy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
||||||
if len(bindings) == 0 {
|
if len(bindings) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -1453,10 +1643,13 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
|||||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||||
withoutDigest := struct {
|
withoutDigest := struct {
|
||||||
ID string
|
ID string
|
||||||
|
ConfiguredValidationPolicy *ValidationPolicyOverride
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
InputExecutionClass contracts.ExecutionClass
|
InputExecutionClass contracts.ExecutionClass
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkExecutionClass contracts.ExecutionClass
|
ChunkExecutionClass contracts.ExecutionClass
|
||||||
|
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
||||||
|
ChunkValidationPolicy ValidationPolicy
|
||||||
ChunkReferences ResolvedReferenceTarget
|
ChunkReferences ResolvedReferenceTarget
|
||||||
Steps []ResolvedPipelineStep
|
Steps []ResolvedPipelineStep
|
||||||
ValidatorChains []ResolvedValidatorChain
|
ValidatorChains []ResolvedValidatorChain
|
||||||
@@ -1464,10 +1657,13 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
|||||||
OutputExecutionClass contracts.ExecutionClass
|
OutputExecutionClass contracts.ExecutionClass
|
||||||
}{
|
}{
|
||||||
ID: resolved.ID,
|
ID: resolved.ID,
|
||||||
|
ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
|
||||||
|
ChunkValidationPolicy: resolved.ChunkValidationPolicy,
|
||||||
Input: resolved.Input,
|
Input: resolved.Input,
|
||||||
InputExecutionClass: resolved.InputExecutionClass,
|
InputExecutionClass: resolved.InputExecutionClass,
|
||||||
Chunk: resolved.Chunk,
|
Chunk: resolved.Chunk,
|
||||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||||
|
ChunkCorrectionProtocol: resolved.ChunkCorrectionProtocol,
|
||||||
ChunkReferences: resolved.ChunkReferences,
|
ChunkReferences: resolved.ChunkReferences,
|
||||||
Steps: resolved.Steps,
|
Steps: resolved.Steps,
|
||||||
ValidatorChains: resolved.ValidatorChains,
|
ValidatorChains: resolved.ValidatorChains,
|
||||||
|
|||||||
@@ -126,6 +126,148 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) {
|
||||||
|
correction := contracts.CorrectionProtocolSingleResponseV1
|
||||||
|
catalog := newProfileCatalogWithOverrides(t,
|
||||||
|
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
|
||||||
|
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}},
|
||||||
|
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||||
|
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||||
|
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||||
|
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||||
|
)
|
||||||
|
resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if resolved.ChunkCorrectionProtocol != correction {
|
||||||
|
t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction)
|
||||||
|
}
|
||||||
|
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||||
|
if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction {
|
||||||
|
t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction)
|
||||||
|
}
|
||||||
|
withoutCapability := cloneResolvedPipeline(resolved)
|
||||||
|
withoutCapability.ChunkCorrectionProtocol = ""
|
||||||
|
withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = ""
|
||||||
|
withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = ""
|
||||||
|
withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = ""
|
||||||
|
withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolvedPipelineDigest() error = %v", err)
|
||||||
|
}
|
||||||
|
if withoutCapabilityDigest == resolved.Digest {
|
||||||
|
t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if prepared.ChunkCorrectionProtocol != correction {
|
||||||
|
t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction)
|
||||||
|
}
|
||||||
|
preparedLane := prepared.Steps[0].ArtifactLanes[0]
|
||||||
|
if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction {
|
||||||
|
t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareRequiresCorrectionCapabilityForValidatorRetries(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
protocol contracts.CorrectionProtocol
|
||||||
|
retries int
|
||||||
|
validators bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "supported", protocol: contracts.CorrectionProtocolSingleResponseV1, retries: 1, validators: true},
|
||||||
|
{name: "unsupported", retries: 1, validators: true, want: "does not declare correction protocol"},
|
||||||
|
{name: "no validators", retries: 1},
|
||||||
|
{name: "no retries", validators: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
catalog := newProfileCatalogWithOverrides(t,
|
||||||
|
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
|
||||||
|
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: test.protocol, Requires: []string{"source"}, Provides: []string{"chunk"}},
|
||||||
|
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||||
|
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||||
|
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||||
|
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||||
|
)
|
||||||
|
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) {
|
||||||
|
return llmProfileTestChunkValidator{key: "chunk-validator"}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := llmProfilePipeline()
|
||||||
|
profile.Chunk.Retries = test.retries
|
||||||
|
if test.validators {
|
||||||
|
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "chunk-validator"}}}
|
||||||
|
}
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
_, err = Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||||
|
if test.want != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type llmProfileTestChunkValidator struct{ key string }
|
||||||
|
|
||||||
|
func (validator llmProfileTestChunkValidator) Name() string { return validator.key }
|
||||||
|
|
||||||
|
func (llmProfileTestChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
|
||||||
|
func (llmProfileTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
class contracts.ExecutionClass
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"},
|
||||||
|
{name: "LLM validator", class: contracts.ExecutionClassLLMBacked},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class})
|
||||||
|
profile := baselineProfile()
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if test.want != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 {
|
||||||
|
t.Fatalf("validator retries = %d, want 1", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
||||||
resolved, err := ResolvePipeline(PipelineProfile{
|
resolved, err := ResolvePipeline(PipelineProfile{
|
||||||
ID: "defaulted",
|
ID: "defaulted",
|
||||||
@@ -408,6 +550,148 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
|
||||||
|
pipelineAttempts := 2
|
||||||
|
chunkAttempts := 1
|
||||||
|
extractAttempts := 0
|
||||||
|
validatorAttempts := 3
|
||||||
|
profile := llmProfilePipeline()
|
||||||
|
profile.StructuredOutputRepairAttempts = &pipelineAttempts
|
||||||
|
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
|
||||||
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(resolved)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(resolved) error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
|
||||||
|
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
|
||||||
|
}
|
||||||
|
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
got *int
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
|
||||||
|
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
|
||||||
|
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if test.got == nil || *test.got != test.want {
|
||||||
|
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
|
||||||
|
if extractChain == nil || len(extractChain.Validators) != 1 {
|
||||||
|
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
|
||||||
|
}
|
||||||
|
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
|
||||||
|
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
|
||||||
|
}
|
||||||
|
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
|
||||||
|
if chunkChain == nil || len(chunkChain.Validators) != 1 {
|
||||||
|
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
|
||||||
|
}
|
||||||
|
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
|
||||||
|
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
pipelineAttempts = 1
|
||||||
|
chunkAttempts = 2
|
||||||
|
extractAttempts = 3
|
||||||
|
validatorAttempts = 0
|
||||||
|
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
|
||||||
|
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*PipelineProfile)
|
||||||
|
}{
|
||||||
|
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
|
||||||
|
{name: "validator", mutate: func(profile *PipelineProfile) {
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
test.mutate(&profile)
|
||||||
|
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.StructuredOutputRepairAttempts = repairAttempts(2)
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
|
||||||
|
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
|
||||||
|
}
|
||||||
|
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||||
|
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
|
||||||
|
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
|
||||||
|
digests := make(map[string]string)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
attempts *int
|
||||||
|
}{
|
||||||
|
{name: "prompt owned", attempts: nil},
|
||||||
|
{name: "disabled", attempts: repairAttempts(0)},
|
||||||
|
{name: "configured", attempts: repairAttempts(1)},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profile := llmProfilePipeline()
|
||||||
|
profile.StructuredOutputRepairAttempts = test.attempts
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
digests[test.name] = resolved.Digest
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
|
||||||
|
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||||
catalog := newProfileCatalog(t)
|
catalog := newProfileCatalog(t)
|
||||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
@@ -1843,6 +2127,10 @@ func llmProfilePipeline() PipelineProfile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func repairAttempts(value int) *int {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func llmProfileValues(profile string) map[string]string {
|
func llmProfileValues(profile string) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
"input": profile,
|
"input": profile,
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ type RunInput struct {
|
|||||||
llmClient contracts.StructuredLLMClient
|
llmClient contracts.StructuredLLMClient
|
||||||
stepID string
|
stepID string
|
||||||
references map[referenceTargetKey]contracts.ReferenceSet
|
references map[referenceTargetKey]contracts.ReferenceSet
|
||||||
|
referenceReuseEligibility map[referenceTargetKey]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type RunOutput struct {
|
type RunOutput struct {
|
||||||
@@ -75,6 +76,9 @@ type RunOutput struct {
|
|||||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||||
OutputFiles []contracts.OutputFile `json:"-"`
|
OutputFiles []contracts.OutputFile `json:"-"`
|
||||||
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
|
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
|
||||||
|
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||||
|
|
||||||
|
normalizeReuseEligibility map[generatedOutputKey]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
|
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
|
||||||
@@ -177,6 +181,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
Path: input.Path,
|
Path: input.Path,
|
||||||
Raw: input.RawInput,
|
Raw: input.RawInput,
|
||||||
LLMProfile: input.pipeline.Input.LLMProfile,
|
LLMProfile: input.pipeline.Input.LLMProfile,
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Input.StructuredOutputRepairAttempts),
|
||||||
Metadata: requestMetadata,
|
Metadata: requestMetadata,
|
||||||
})
|
})
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
@@ -240,6 +245,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
if chunkResult.rejection != nil {
|
if chunkResult.rejection != nil {
|
||||||
output.Rejected = append(output.Rejected, *chunkResult.rejection)
|
output.Rejected = append(output.Rejected, *chunkResult.rejection)
|
||||||
}
|
}
|
||||||
|
if chunkResult.validation != nil {
|
||||||
|
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(*chunkResult.validation))
|
||||||
|
}
|
||||||
output.Warnings = append(output.Warnings, chunkResult.warnings...)
|
output.Warnings = append(output.Warnings, chunkResult.warnings...)
|
||||||
chunkDebugPayload := map[string]any{
|
chunkDebugPayload := map[string]any{
|
||||||
"cache_mode": chunkMode,
|
"cache_mode": chunkMode,
|
||||||
@@ -296,6 +304,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
|
|
||||||
if len(output.Rejected) > 0 {
|
if len(output.Rejected) > 0 {
|
||||||
output.Manifest.ValidationStatus = "rejected"
|
output.Manifest.ValidationStatus = "rejected"
|
||||||
|
} else if hasIncompleteValidation(output.ValidationSummaries) {
|
||||||
|
output.Manifest.ValidationStatus = "incomplete"
|
||||||
} else {
|
} else {
|
||||||
output.Manifest.ValidationStatus = "approved"
|
output.Manifest.ValidationStatus = "approved"
|
||||||
}
|
}
|
||||||
@@ -365,6 +375,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
Rejected: cloneRejectedOutputs(output.Rejected),
|
Rejected: cloneRejectedOutputs(output.Rejected),
|
||||||
Warnings: output.Warnings,
|
Warnings: output.Warnings,
|
||||||
LLMProfile: input.pipeline.Output.LLMProfile,
|
LLMProfile: input.pipeline.Output.LLMProfile,
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Output.StructuredOutputRepairAttempts),
|
||||||
Metadata: outputMetadata,
|
Metadata: outputMetadata,
|
||||||
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
||||||
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
||||||
@@ -399,6 +410,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func hasIncompleteValidation(summaries []artifacts.ValidationSummary) bool {
|
||||||
|
for _, summary := range summaries {
|
||||||
|
if summary.Status == "incomplete" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
|
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
|
||||||
for _, step := range input.Prepared.Steps {
|
for _, step := range input.Prepared.Steps {
|
||||||
stepInput := input
|
stepInput := input
|
||||||
@@ -408,6 +428,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
|
|||||||
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
|
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
|
||||||
}
|
}
|
||||||
stepInput.references = stepReferences
|
stepInput.references = stepReferences
|
||||||
|
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
|
||||||
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
|
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
|
||||||
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
|
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
|
||||||
if err := mergeLaneOutput(output, laneOutput); err != nil {
|
if err := mergeLaneOutput(output, laneOutput); err != nil {
|
||||||
@@ -426,7 +447,7 @@ type retryAttemptResult struct {
|
|||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
}
|
}
|
||||||
|
|
||||||
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (retryAttemptResult, error)) (retryAttemptResult, error) {
|
func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (retryAttemptResult, error)) (retryAttemptResult, error) {
|
||||||
attempts := retries + 1
|
attempts := retries + 1
|
||||||
var last retryAttemptResult
|
var last retryAttemptResult
|
||||||
for attempt := 1; attempt <= attempts; attempt++ {
|
for attempt := 1; attempt <= attempts; attempt++ {
|
||||||
@@ -468,34 +489,36 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (retry
|
|||||||
return last, nil
|
return last, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
|
||||||
content, err := json.Marshal(chunks)
|
content, err := json.Marshal(chunks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
|
return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err)
|
||||||
}
|
}
|
||||||
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
||||||
var warnings []contracts.Warning
|
report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
|
||||||
for index, item := range prepared.validators {
|
|
||||||
binding := item.resolved.Binding
|
binding := item.resolved.Binding
|
||||||
started := time.Now().UTC()
|
started := time.Now().UTC()
|
||||||
attemptPath := path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, fileio.EncodePathComponent(binding.Module), attempt))
|
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
|
||||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(ctx, attemptPath)
|
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
||||||
var result contracts.ValidationResult
|
var result contracts.ValidationResult
|
||||||
requestMetadata, cloneErr := cloneMetadata(metadata)
|
requestMetadata, cloneErr := cloneMetadata(metadata)
|
||||||
if cloneErr != nil {
|
if cloneErr != nil {
|
||||||
return nil, nil, fmt.Errorf("clone chunk validation metadata: %w", cloneErr)
|
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
|
||||||
}
|
}
|
||||||
requestChunks, cloneErr := cloneSourceChunks(chunks)
|
requestChunks, cloneErr := cloneSourceChunks(chunks)
|
||||||
if cloneErr != nil {
|
if cloneErr != nil {
|
||||||
return nil, nil, fmt.Errorf("clone chunks for validation: %w", cloneErr)
|
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
|
||||||
}
|
}
|
||||||
switch item.resolved.Target {
|
switch item.resolved.Target {
|
||||||
case ValidatorTargetChunk:
|
case ValidatorTargetChunk:
|
||||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks})
|
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
|
||||||
case ValidatorTargetSerialized:
|
case ValidatorTargetSerialized:
|
||||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
||||||
default:
|
default:
|
||||||
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
|
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = contracts.ValidateValidationResult(result)
|
||||||
}
|
}
|
||||||
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
|
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
|
||||||
debugContent.ContentDigest = debugContentDigest(content)
|
debugContent.ContentDigest = debugContentDigest(content)
|
||||||
@@ -506,27 +529,19 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
|
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
||||||
return warnings, nil, errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
|
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
|
||||||
}
|
}
|
||||||
return warnings, nil, validationErr
|
return validationInvocation{}, validationErr
|
||||||
}
|
}
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
||||||
return warnings, nil, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)
|
return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
|
||||||
}
|
}
|
||||||
if !result.Approved {
|
return validationInvocation{result: result}, nil
|
||||||
reason := result.ReasonCode
|
})
|
||||||
if reason == "" {
|
if err != nil {
|
||||||
reason = "output_rejected"
|
return report, err
|
||||||
}
|
}
|
||||||
message := result.Message
|
return report, nil
|
||||||
if message == "" {
|
|
||||||
message = "output rejected"
|
|
||||||
}
|
|
||||||
return warnings, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
|
|
||||||
}
|
|
||||||
warnings = append(warnings, result.Warnings...)
|
|
||||||
}
|
|
||||||
return warnings, nil, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||||
@@ -724,6 +739,9 @@ func populateOutputManifest(output *RunOutput) {
|
|||||||
}
|
}
|
||||||
output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
|
output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
|
||||||
output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
|
output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
|
||||||
|
if len(output.ValidationSummaries) > 0 {
|
||||||
|
output.Manifest.ValidationSummaries = cloneValidationSummaries(output.ValidationSummaries)
|
||||||
|
}
|
||||||
if len(output.CheckpointEvents) > 0 {
|
if len(output.CheckpointEvents) > 0 {
|
||||||
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
|
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
|
||||||
for _, event := range output.CheckpointEvents {
|
for _, event := range output.CheckpointEvents {
|
||||||
@@ -773,6 +791,7 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
|
|||||||
Message: output.Message,
|
Message: output.Message,
|
||||||
AttemptCount: output.AttemptCount,
|
AttemptCount: output.AttemptCount,
|
||||||
DiagnosticArtifactPath: output.DiagnosticArtifactPath,
|
DiagnosticArtifactPath: output.DiagnosticArtifactPath,
|
||||||
|
Validation: cloneValidationSummaryPtr(output.Validation),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return manifests
|
return manifests
|
||||||
@@ -1031,7 +1050,31 @@ func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.Rejec
|
|||||||
if len(rejected) == 0 {
|
if len(rejected) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
cloned := make([]contracts.RejectedOutput, len(rejected))
|
||||||
|
for index, item := range rejected {
|
||||||
|
cloned[index] = item
|
||||||
|
cloned[index].Validation = cloneValidationSummaryPtr(item.Validation)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidationSummaryPtr(summary *artifacts.ValidationSummary) *artifacts.ValidationSummary {
|
||||||
|
if summary == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := artifacts.CloneValidationSummary(*summary)
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
||||||
|
if len(summaries) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
||||||
|
for index, summary := range summaries {
|
||||||
|
cloned[index] = artifacts.CloneValidationSummary(summary)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
func timePtr(t time.Time) *time.Time {
|
func timePtr(t time.Time) *time.Time {
|
||||||
|
|||||||
@@ -117,6 +117,38 @@ func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
|
|||||||
return prepared
|
return prepared
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
|
||||||
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
|
lane := &prepared.Steps[0].lanes[0]
|
||||||
|
producerAttempts := 2
|
||||||
|
validatorAttempts := 0
|
||||||
|
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
|
||||||
|
if len(lane.extractValidators.validators) == 0 {
|
||||||
|
t.Fatal("extract validators are empty")
|
||||||
|
}
|
||||||
|
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
|
||||||
|
|
||||||
|
var observedProducer, observedValidator *int
|
||||||
|
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||||
|
observedProducer = request.StructuredOutputRepairAttempts
|
||||||
|
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
|
||||||
|
})
|
||||||
|
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
|
observedValidator = target.structuredOutputRepairAttempts
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
|
||||||
|
}
|
||||||
|
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
||||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
||||||
return err
|
return err
|
||||||
@@ -198,14 +230,14 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
|
|||||||
if attempts == 1 {
|
if attempts == 1 {
|
||||||
scope = "discarded"
|
scope = "discarded"
|
||||||
}
|
}
|
||||||
return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Warnings: []contracts.Warning{{Scope: scope, ReasonCode: "observed", Message: scope}}}, nil
|
return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Warnings: []contracts.Warning{{Scope: scope, ReasonCode: "observed", Message: scope}}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope))}, nil
|
||||||
}
|
}
|
||||||
validatorCalls := 0
|
validatorCalls := 0
|
||||||
validator := preparedValidator{
|
validator := preparedValidator{
|
||||||
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
validatorCalls++
|
validatorCalls++
|
||||||
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate"}, nil
|
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
switch stage {
|
switch stage {
|
||||||
@@ -253,6 +285,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
|||||||
path string
|
path string
|
||||||
wantError string
|
wantError string
|
||||||
wantBody string
|
wantBody string
|
||||||
|
attemptError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "merge module error",
|
name: "merge module error",
|
||||||
@@ -263,10 +296,12 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
|||||||
},
|
},
|
||||||
path: "merge/notes/attempt-01.json",
|
path: "merge/notes/attempt-01.json",
|
||||||
wantError: "merge exploded",
|
wantError: "merge exploded",
|
||||||
|
attemptError: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "normalize validator error",
|
name: "normalize validator error",
|
||||||
configure: func(prepared *PreparedPipeline) {
|
configure: func(prepared *PreparedPipeline) {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureFailRun
|
||||||
prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{
|
prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{
|
||||||
resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
@@ -281,10 +316,11 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "merge final rejection",
|
name: "merge final rejection",
|
||||||
configure: func(prepared *PreparedPipeline) {
|
configure: func(prepared *PreparedPipeline) {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||||
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
|
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
},
|
},
|
||||||
@@ -300,6 +336,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
|||||||
},
|
},
|
||||||
path: "normalize/notes/attempt-01.json",
|
path: "normalize/notes/attempt-01.json",
|
||||||
wantError: "serialize normalize candidate",
|
wantError: "serialize normalize candidate",
|
||||||
|
attemptError: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -311,9 +348,15 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
|||||||
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||||
envelope := debug.envelope(t, tc.path)
|
envelope := debug.envelope(t, tc.path)
|
||||||
if tc.wantError != "" {
|
if tc.wantError != "" {
|
||||||
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
|
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
|
||||||
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
|
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
|
||||||
}
|
}
|
||||||
|
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||||
|
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
|
||||||
|
}
|
||||||
|
if !tc.attemptError && envelope.Error != "" {
|
||||||
|
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||||
|
}
|
||||||
} else if runErr != nil {
|
} else if runErr != nil {
|
||||||
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
|
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
|
|||||||
validator := preparedValidator{
|
validator := preparedValidator{
|
||||||
resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator"}, nil
|
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
switch target {
|
switch target {
|
||||||
@@ -187,6 +187,11 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
|
|||||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||||
t.Run(string(target), func(t *testing.T) {
|
t.Run(string(target), func(t *testing.T) {
|
||||||
prepared := preparedAttemptDebugPipeline(t)
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
|
if target == StageMerge {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
|
} else {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
|
}
|
||||||
prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{}
|
prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{}
|
||||||
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
|
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
|
||||||
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
|
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
|
||||||
@@ -306,6 +311,11 @@ func TestRunnerRejectsCandidatesBeforeFinalEncoding(t *testing.T) {
|
|||||||
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
|
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
|
||||||
t.Run(string(target), func(t *testing.T) {
|
t.Run(string(target), func(t *testing.T) {
|
||||||
prepared := preparedAttemptDebugPipeline(t)
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
|
if target == StageMerge {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
|
} else {
|
||||||
|
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||||
|
}
|
||||||
codec := &observedNotesCodec{}
|
codec := &observedNotesCodec{}
|
||||||
installObservedNotesCodec(t, prepared, codec)
|
installObservedNotesCodec(t, prepared, codec)
|
||||||
configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}})
|
configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}})
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user