Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 916d9210fd | |||
| 3d3f16db4a | |||
| 63c397d86a | |||
| 9a92212632 | |||
| 0ef8931697 | |||
| e00cc45c6b | |||
| ab9b743df6 | |||
| 3bd3c7ebf7 | |||
| 75a3f51cee | |||
| 2be999ebd3 | |||
| 32fe7c5b98 | |||
| 55247c47ab | |||
| 5e5c69bf9d | |||
| b4a81f8b09 | |||
| adfed22e1a | |||
| 75b1e2f68b | |||
| 4473363d9f | |||
| 6eb45e0003 | |||
| 0585ad76dc | |||
| 56145c3b7e | |||
| a478fd86c5 | |||
| 916532100d |
33
.woodpecker/release.yml
Normal file
33
.woodpecker/release.yml
Normal file
@@ -0,0 +1,33 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: validate-release
|
||||
image: golang:1.25.5
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
release_note="docs/releases/$version.md"
|
||||
|
||||
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
|
||||
printf '%s\n' "invalid release tag: $version" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$release_note" ]; then
|
||||
printf '%s\n' "missing release note: $release_note" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
|
||||
printf '%s\n' "release note heading does not match $version" >&2
|
||||
exit 1
|
||||
fi
|
||||
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
|
||||
printf '%s\n' "release note is missing heading: $heading" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
./scripts/check-release-source.sh "$version"
|
||||
16
README.md
16
README.md
@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
|
||||
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
||||
demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||
|
||||
## Install A Source Release
|
||||
|
||||
Install a pinned source release with Go:
|
||||
|
||||
~~~
|
||||
GOWORK=off go install \
|
||||
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
|
||||
~~~
|
||||
|
||||
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
|
||||
installed command's diagnostic version is described in the [CLI
|
||||
reference](docs/cli.md); maintainers preparing a release should follow [Source
|
||||
Releases](docs/release.md).
|
||||
|
||||
## Documentation
|
||||
|
||||
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
||||
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||
artifact formats.
|
||||
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
||||
from an orchestrator and consume a published result.
|
||||
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
|
||||
D&D pipeline as a subprocess and discover its structured artifacts.
|
||||
- [Internal overview](docs/internal/overview.md) — implemented component map
|
||||
for maintainers.
|
||||
- [Developer guide](docs/development.md) — contributor orientation and
|
||||
|
||||
@@ -42,4 +42,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_combat_turns_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -50,4 +50,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_enemy_events_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -42,4 +42,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_occurrences_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -37,4 +37,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_registry_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -27,4 +27,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -42,4 +42,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_occurrences_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -37,4 +37,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_registry_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -27,4 +27,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -42,4 +42,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -37,4 +37,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_registry_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -27,4 +27,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -35,4 +35,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -31,4 +31,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scenes_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -47,4 +47,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_spells_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -24,4 +24,4 @@ output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 0
|
||||
repair_attempts: 1
|
||||
|
||||
12
docs/cli.md
12
docs/cli.md
@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
|
||||
|
||||
~~~
|
||||
notarius help
|
||||
notarius --version
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
|
||||
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
||||
writes the command summary to standard output and exits with status 0.
|
||||
|
||||
`notarius --version` is valid only as the sole root argument. It writes exactly
|
||||
`notarius <version>` followed by a newline to standard output and exits with
|
||||
status 0. A tagged `go install` build can report its main-module stable tag,
|
||||
and controlled builds can inject a stable tag at link time through
|
||||
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
|
||||
unversioned checkout reports `development`. Invalid injected version content is
|
||||
a runtime error with exit status 1, while extra `--version` arguments are a
|
||||
syntax error with exit status 2. This diagnostic does not replace the
|
||||
[run-result](integrations/run-result.md) or artifact contracts for downstream
|
||||
compatibility decisions.
|
||||
|
||||
## run
|
||||
|
||||
~~~
|
||||
|
||||
@@ -132,7 +132,11 @@ model: example-model
|
||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||
its credential environment variable through `api_key_env`; set that variable
|
||||
only in the run environment. PromptKit owns the
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md),
|
||||
including `base_profile` inheritance. Notarius passes profiles through without
|
||||
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||
provider without authorization.
|
||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||
supported package API, and [Operations](operations.md#operational-limits)
|
||||
describes the effective concurrency layers.
|
||||
@@ -218,6 +222,7 @@ pipelines:
|
||||
| Field | Type | Default | Rules |
|
||||
| --- | --- | --- | --- |
|
||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
|
||||
| **input** | module binding | none | Required. |
|
||||
| **chunk** | module binding | **generic** | Optional. |
|
||||
| **output** | module binding | **json** | Optional. |
|
||||
@@ -237,6 +242,15 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
|
||||
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
||||
Deterministic bindings do not receive these defaults or run overrides.
|
||||
|
||||
Structural output repair is resolved after module, validator, and `--only` lane
|
||||
selection. An object's **structured_output_repair_attempts** value takes
|
||||
precedence over the pipeline value; otherwise, an LLM-backed binding or
|
||||
validator inherits the pipeline value. If both are omitted, PromptKit uses the
|
||||
prompt's declared repair policy. The value must be an integer from 0 through 3;
|
||||
explicit `null` and non-integer values are invalid. An explicit value on a
|
||||
deterministic binding or validator is invalid, while a pipeline value simply
|
||||
does not apply to deterministic selections.
|
||||
|
||||
A lane has these fields:
|
||||
|
||||
| Field | Type | Default | Rules |
|
||||
@@ -274,6 +288,7 @@ extract:
|
||||
| --- | --- | --- | --- |
|
||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
|
||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
||||
| **options** | object | none | Must satisfy the selected module. |
|
||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||
@@ -281,10 +296,11 @@ extract:
|
||||
|
||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||
Validator bindings accept only **module**, **llm_profile**, and **options**.
|
||||
They reject **references**, **retries**, and nested **validators**. Deterministic
|
||||
validators reject an explicit **llm_profile**. Deterministic module bindings
|
||||
also reject an explicit **llm_profile**.
|
||||
Validator bindings accept only **module**, **llm_profile**,
|
||||
**structured_output_repair_attempts**, and **options**. They reject
|
||||
**references**, **retries**, and nested **validators**. Deterministic validators
|
||||
reject explicit **llm_profile** and **structured_output_repair_attempts**.
|
||||
Deterministic module bindings also reject those explicit fields.
|
||||
|
||||
The **json** output module accepts optional **include_chunk_map** and
|
||||
**evidence_context** settings:
|
||||
|
||||
202
docs/consumers/dnd-pipeline.md
Normal file
202
docs/consumers/dnd-pipeline.md
Normal file
@@ -0,0 +1,202 @@
|
||||
# Consuming The Complete D&D Pipeline
|
||||
|
||||
Use this workflow when an orchestrator runs the maintained complete D&D
|
||||
pipeline and consumes its structured JSON artifacts. The generic
|
||||
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
|
||||
this guide connects that workflow to the complete D&D configuration, its
|
||||
Seriatim input, and its artifact inventory.
|
||||
|
||||
The [CLI reference](../cli.md), [configuration reference](../config.md),
|
||||
[run-result receipt](../integrations/run-result.md), and
|
||||
[published JSON output contract](../integrations/json-output.md) remain the
|
||||
canonical definitions of those public interfaces.
|
||||
|
||||
## Prepare And Validate The Deployment
|
||||
|
||||
Start from the maintained
|
||||
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
|
||||
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
|
||||
artifact handoffs, campaign references, chunk-map publication, and evidence
|
||||
context.
|
||||
|
||||
A deployment must provide its own PromptKit profile and campaign reference
|
||||
files. Use absolute paths for service and subprocess deployments. In
|
||||
particular, observe these different resolution rules:
|
||||
|
||||
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||
file; and
|
||||
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||
directory.
|
||||
|
||||
Do not copy the repository example's relative profile path into a deployment
|
||||
without also controlling that working directory. The complete path and profile
|
||||
rules are defined in [Configuration](../config.md).
|
||||
|
||||
Preflight the deployed configuration before processing sessions and whenever
|
||||
it changes:
|
||||
|
||||
```sh
|
||||
notarius config validate \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--pipeline dnd-session
|
||||
```
|
||||
|
||||
Provide credentials through the environment or the documented configuration
|
||||
mechanism. Do not put credentials in command arguments, generated
|
||||
configuration, or logs.
|
||||
|
||||
## Supply The Transcript
|
||||
|
||||
The complete pipeline consumes a Seriatim JSON document. The
|
||||
[Seriatim input contract](../integrations/seriatim.md) defines its required
|
||||
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
|
||||
citations use those segment IDs as source-unit ranges.
|
||||
|
||||
When the caller maintains several transcript tiers, use the final trimmed JSON
|
||||
transcript so extraction operates on the same session content presented to
|
||||
later consumers. For example, Narratio identifies this implemented artifact as
|
||||
`narratio.transcript.final_trimmed` and normally stores it at
|
||||
`transcripts/final.trimmed.json`.
|
||||
|
||||
Notarius generates a stable prompt session from the resolved input module and
|
||||
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
|
||||
Use that override only when intentionally changing the routing relationship
|
||||
between invocations; it is not a credential or output identity.
|
||||
|
||||
## Run Notarius
|
||||
|
||||
Invoke the pipeline with explicit absolute paths and request its
|
||||
machine-readable receipt:
|
||||
|
||||
```sh
|
||||
notarius run dnd-session \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||
--output-dir /absolute/path/to/notarius-output \
|
||||
--json
|
||||
```
|
||||
|
||||
The caller should:
|
||||
|
||||
- capture stdout and stderr separately;
|
||||
- propagate cancellation and impose an operator-appropriate timeout;
|
||||
- wait for process completion before interpreting stdout; and
|
||||
- retain stderr for diagnosis without copying secrets or transcript content
|
||||
into other logs.
|
||||
|
||||
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
|
||||
nonzero exit because a failed receipt write can leave partial bytes. The
|
||||
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
|
||||
stream and exit-status contract.
|
||||
|
||||
## Discover The Published Bundle
|
||||
|
||||
Decode the successful stdout document as a supported run-result schema. For
|
||||
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate
|
||||
unknown fields allowed by that version, but reject an unsupported schema
|
||||
version.
|
||||
|
||||
Use the receipt's absolute `output_directory` as the exact run-specific bundle
|
||||
root. Do not scan the output root for its newest directory, guess a run ID, or
|
||||
construct a bundle path. Resolve `index_file` beneath `output_directory` and
|
||||
reject an absolute logical path or any result that escapes the bundle root.
|
||||
|
||||
Read `index.json` and locate each requested lane in `output_files` by its exact
|
||||
`lane_id`. Do not guess a lane filename. Before decoding a payload:
|
||||
|
||||
1. resolve its descriptor's relative `file` beneath the bundle root with the
|
||||
same confinement check;
|
||||
2. verify the descriptor's media type and schema identity against the linked
|
||||
artifact contract; and
|
||||
3. decode the payload according to that contract.
|
||||
|
||||
The [published JSON output contract](../integrations/json-output.md) defines
|
||||
the index and bundle layout. Treat all paths obtained from a decoded external
|
||||
document as untrusted until confined to their documented root.
|
||||
|
||||
## Complete Artifact Inventory
|
||||
|
||||
When every configured lane is accepted, the complete example publishes these
|
||||
lane artifacts:
|
||||
|
||||
| Lane ID | Purpose | Canonical contract |
|
||||
| --- | --- | --- |
|
||||
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
|
||||
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
|
||||
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
|
||||
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
|
||||
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
|
||||
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
|
||||
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
|
||||
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
|
||||
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
|
||||
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
|
||||
|
||||
The JSON encoder always publishes these bundle-management files:
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
|
||||
| `manifest.json` | Run provenance and result summaries. |
|
||||
| `rejected.json` | Rejected pipeline outputs. |
|
||||
| `warnings.json` | Accepted-output and run warnings. |
|
||||
|
||||
The complete configuration also requests two pipeline-wide artifacts:
|
||||
|
||||
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
|
||||
chunk metadata; and
|
||||
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
|
||||
excerpt containing the union of selected cited source units and the
|
||||
configured surrounding window.
|
||||
|
||||
Discover both from their top-level `index.json` descriptors rather than
|
||||
treating them as lanes. Evidence context is convenient reading material, not
|
||||
authoritative provenance; citations in the normalized lane payloads remain the
|
||||
evidence contract.
|
||||
|
||||
Every optional or lane file is published only when its corresponding artifact
|
||||
is available. A successful process does not guarantee that all configured
|
||||
lanes were accepted.
|
||||
|
||||
## Decide What Counts As Consumer Success
|
||||
|
||||
Exit status 0 means Notarius completed the pipeline and published its result
|
||||
bundle. The receipt or bundle may still report warnings, rejected outputs, or
|
||||
missing lane descriptors. A downstream consumer must define its own required
|
||||
artifact set explicitly.
|
||||
|
||||
A caller that claims to consume the complete D&D workflow should normally
|
||||
require all ten lane IDs in the table and verify each descriptor's expected
|
||||
contract. If any required lane is missing, rejected, or incompatible, fail the
|
||||
caller's extraction step while retaining the Notarius bundle for diagnosis. A
|
||||
consumer that needs only a subset may define and document a narrower policy.
|
||||
|
||||
Keep the successful receipt with the complete published bundle. Retain
|
||||
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
|
||||
required by the caller's provenance, diagnosis, and retention policies. Avoid
|
||||
selectively copying payload files without also preserving enough index and
|
||||
manifest information to identify their originating run and contracts.
|
||||
|
||||
The transcript, lane artifacts, evidence context, manifest, debug data, and
|
||||
logs can all contain private campaign information. Apply the same access,
|
||||
publication, and retention controls used for the source transcript.
|
||||
|
||||
## Consumer Checklist
|
||||
|
||||
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
|
||||
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
|
||||
- Use absolute configuration, input, output-root, profile, and reference paths
|
||||
in service deployments.
|
||||
- Capture stdout and stderr separately and enforce cancellation and timeout.
|
||||
- Parse stdout only after exit status 0.
|
||||
- Accept only supported receipt, index, and artifact schema versions while
|
||||
tolerating permitted unknown fields.
|
||||
- Use the receipt's `output_directory`; never guess the run directory.
|
||||
- Confine `index_file` and every descriptor path to the published bundle root.
|
||||
- Discover lanes by `lane_id` and verify descriptor compatibility before
|
||||
decoding payloads.
|
||||
- Enforce an explicit required-lane policy and inspect rejections and warnings.
|
||||
- Preserve the receipt and sufficient bundle provenance for every retained
|
||||
artifact.
|
||||
- Protect all transcript-derived files and diagnostic streams as sensitive
|
||||
campaign data.
|
||||
@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
|
||||
[Published JSON Output contract](../integrations/json-output.md) own the
|
||||
durable result formats.
|
||||
|
||||
For the maintained complete D&D workflow, including its transcript input,
|
||||
configured lane inventory, and downstream acceptance checklist, see
|
||||
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
|
||||
|
||||
## Run And Check The Process
|
||||
|
||||
Optionally preflight a selected configuration and pipeline before work starts:
|
||||
|
||||
@@ -18,13 +18,14 @@ implemented component map.
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
||||
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
|
||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
|
||||
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
||||
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
|
||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
For an existing subsystem, also inspect its focused tests and the package-local
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# PromptKit Integration
|
||||
|
||||
Notarius pins
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0)
|
||||
as its in-process prompt engine. The upstream
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/consumers/pkg-promptkit.md)
|
||||
owns the public engine API, and the upstream
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md)
|
||||
owns prompt, profile, and schema file contracts.
|
||||
|
||||
## Supported Boundary
|
||||
@@ -26,7 +26,7 @@ Notarius relies on the root `promptkit` package to:
|
||||
admission exhaustion through `ErrCapacityExceeded`.
|
||||
|
||||
The pinned
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/backends.go)
|
||||
owns the registration and backend-capacity contract.
|
||||
|
||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||
@@ -52,7 +52,7 @@ Notarius sends one stable effective session through PromptKit's direct session
|
||||
field, which is authoritative for provider session behavior. It also retains
|
||||
the same value as the `session_id` prompt variable for maintained prompt
|
||||
compatibility. The generated identifier is 76 ASCII characters, within
|
||||
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret
|
||||
PromptKit v0.8.0's 256-code-point session limit. Session IDs are non-secret
|
||||
correlation identifiers and may be exposed to providers and provider
|
||||
observability. The CLI contract owns generation and override behavior.
|
||||
|
||||
@@ -84,7 +84,13 @@ configuration and deployment workflow are defined in
|
||||
[Configuration](../config.md#promptkit-profiles) and
|
||||
[Operations](../operations.md#promptkit-profile-deployment).
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
|
||||
PromptKit owns `base_profile` resolution under its
|
||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md).
|
||||
Notarius records the selected leaf identity and resolved target without parsing
|
||||
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||
reach the provider without authorization, which can result in a 401 or 403.
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
|
||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||
@@ -97,6 +103,10 @@ pinned upstream documentation.
|
||||
module assets, maps its transport-neutral completion contract, prepares and
|
||||
executes requests, validates output, records provenance, captures debug
|
||||
material, redacts errors, and preserves timeout ownership.
|
||||
|
||||
PromptKit provider error details do not cross the ordinary completion boundary.
|
||||
Notarius exposes a provider-neutral generation category and optional status;
|
||||
redacted provider details are retained only in requested debug material.
|
||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||
@@ -105,4 +115,7 @@ the conventional local backend.
|
||||
|
||||
PromptKit API or format changes outside this boundary are not implicitly
|
||||
supported. Updating the pinned version requires reviewing the adapter and
|
||||
profile/configuration contracts against the upstream documentation.
|
||||
profile/configuration contracts against the upstream documentation. Maintained
|
||||
production prompts use PromptKit's bounded structural-repair contract; their
|
||||
current declaration is one additional repair attempt. Notarius retains the
|
||||
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||
|
||||
@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
|
||||
|
||||
## Dispatch And Configuration Handoff
|
||||
|
||||
The root dispatcher handles help, configuration validation, pipeline listing,
|
||||
and a pipeline run. It normalizes injectable options before dispatch so that a
|
||||
missing production dependency fails as a command error rather than reaching
|
||||
execution.
|
||||
The root dispatcher handles help, version reporting, configuration validation,
|
||||
pipeline listing, and a pipeline run. Version reporting resolves build
|
||||
information through `internal/buildinfo` before production composition, so the
|
||||
diagnostic remains available without configuration or runtime collaborators.
|
||||
The public syntax, streams, exit classes, and version semantics are defined by
|
||||
the [CLI reference](../cli.md). Other root commands normalize injectable
|
||||
options before dispatch so that a missing production dependency fails as a
|
||||
command error rather than reaching execution.
|
||||
|
||||
Commands that need configuration use one shared loader. The CLI discovers the
|
||||
file, parses it through **internal/core/config**, starts from defaults, applies
|
||||
|
||||
@@ -75,7 +75,7 @@ backend membership as runtime without performing generation. Fallback assets
|
||||
are mounted only when at least one source is registered. The production D&D
|
||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||
profile parsing: an operator-provided matching profile takes precedence over a
|
||||
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||
fallback profile without Notarius merging either document.
|
||||
When the registration is absent, a profile selecting `backend: local` fails
|
||||
inspection instead of falling back to a built-in or endpoint-only target.
|
||||
@@ -210,14 +210,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
|
||||
the pipeline's existing binding attempt policy sees the operational error and
|
||||
decides whether to rerun the complete operation.
|
||||
|
||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
||||
The current production D&D prompt manifests set repair attempts to zero. That
|
||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
||||
rejection is a recorded output rather than a provider error. The pipeline owns
|
||||
attempt lifecycle, validation chains, and retry diagnostics; see
|
||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||
[binding reference](../config.md#module-bindings-and-validators).
|
||||
PromptKit executes structural repair within its structured-output flow. The
|
||||
maintained production prompt manifests declare one additional repair attempt.
|
||||
When a resolved binding supplies a repair value, the adapter inspects the
|
||||
prompt, copies its complete output contract, changes only the repair limit, and
|
||||
passes that complete replacement contract to PromptKit. This preserves the
|
||||
prompt's output format, validation mode, schema, and provider structured-output
|
||||
settings.
|
||||
|
||||
A successful repair is an ordinary successful completion, not a warning. The
|
||||
adapter reports PromptKit's actual repair count and its cumulative usage
|
||||
directly, without adding the initial and corrective counts again. Debug prompt
|
||||
material records the configured complete contract; debug response material
|
||||
records the repaired response and actual validation result. If the repair
|
||||
budget is exhausted, the adapter retains the final raw bytes and debug material
|
||||
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
|
||||
or corrective call remain provider-neutral operational errors with the same
|
||||
redaction boundary.
|
||||
|
||||
Structural repair does not replace pipeline retry behavior: a binding's
|
||||
configured retry count reruns its complete stage attempt after an error or
|
||||
rejection. The pipeline owns attempt lifecycle, validation chains, and retry
|
||||
diagnostics; see [Pipeline Internals](pipeline.md#validation-retries-and-output)
|
||||
and the [binding reference](../config.md#module-bindings-and-validators).
|
||||
|
||||
## Timeout Ownership
|
||||
|
||||
@@ -256,6 +271,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
|
||||
available to other runtime collaborators; it does not make prompt or response
|
||||
contents safe for general logging.
|
||||
|
||||
Generation failures expose an application-owned category and optional HTTP
|
||||
status. Provider code, type, and message remain debug-only, after redaction.
|
||||
|
||||
## Failure Boundaries
|
||||
|
||||
- Construction fails for missing asset registries, mutually exclusive profile
|
||||
|
||||
@@ -25,6 +25,7 @@ physical state roots.
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
||||
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
|
||||
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
||||
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
||||
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
||||
|
||||
@@ -36,9 +36,16 @@ assigns a deterministic resolved-composition digest. The resolved pipeline
|
||||
contains bindings and declared reference targets, not external reference bytes.
|
||||
After selection, the resolver applies command, binding, and pipeline profile
|
||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||
an empty resolved binding profile. Deterministic bindings remain profile-free.
|
||||
These effective values are part of the digest, so execution and checkpoint
|
||||
consumers do not repeat profile inheritance.
|
||||
an empty resolved binding profile. It resolves structural output repair
|
||||
separately: a binding's `structured_output_repair_attempts` value wins, then a
|
||||
pipeline value applies to LLM-backed bindings and validators, and omission
|
||||
leaves the prompt-owned policy intact. An explicit repair value on a
|
||||
deterministic binding is rejected. Resolved bindings own copied repair values,
|
||||
and these effective values are part of the digest, so execution and checkpoint
|
||||
consumers do not repeat profile inheritance or configuration resolution.
|
||||
Each LLM request receives its own copy of that resolved value. PromptKit spends
|
||||
it only for structural correction inside one completion; the runner's binding
|
||||
retry policy remains the separate outer budget for complete stage attempts.
|
||||
Configuration resolution supplies the selected profile and catalog; see
|
||||
[Configuration Internals](configuration.md).
|
||||
|
||||
|
||||
@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
|
||||
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
||||
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
||||
|
||||
## Source Deployment
|
||||
|
||||
Linux is the supported deployment platform. Install a pinned source release
|
||||
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
||||
|
||||
~~~sh
|
||||
GOWORK=off go install \
|
||||
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||
~~~
|
||||
|
||||
Pin the exact tag in deployment automation rather than following a branch.
|
||||
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
|
||||
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
|
||||
The maintainer publication process, including tag guards and verification,
|
||||
belongs to [Source Releases](release.md). macOS builds are best-effort for
|
||||
development, and Windows is unsupported.
|
||||
|
||||
## State Surfaces
|
||||
|
||||
Each run can use independent roots with different retention and access-control
|
||||
@@ -72,6 +89,9 @@ provider call or credentials:
|
||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||
~~~
|
||||
|
||||
An unset optional `api_key_env` reaches the provider without authorization and
|
||||
may receive a 401 or 403 response.
|
||||
|
||||
Profile paths are currently resolved from the process working directory, not
|
||||
from the configuration file. The complete example's
|
||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||
@@ -255,14 +275,29 @@ Provider execution settings and the generation timeout come from the selected
|
||||
PromptKit profile. The invocation-only **--reasoning-effort** and
|
||||
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
||||
for all LLM-backed calls in one run without changing the profile. PromptKit
|
||||
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
|
||||
complete module operation and validation chain as defined by
|
||||
[module bindings](config.md#module-bindings-and-validators).
|
||||
structural output repair happens within one structured-completion call. Its
|
||||
effective `structured_output_repair_attempts` limit is resolved from the
|
||||
selected binding, then the pipeline, then the prompt declaration; see
|
||||
[module bindings](config.md#module-bindings-and-validators). This is distinct
|
||||
from Notarius binding **retries**, which rerun the complete module operation
|
||||
and validation chain and do not consume or replenish the structural-repair
|
||||
limit. The maintained production prompts declare one repair attempt, paid only
|
||||
after a structural failure. One structured completion with repair budget **R**
|
||||
makes at most **R + 1** serial provider calls. If one stage attempt performs
|
||||
**C** structured completions, a binding with **retries: N** has a maximum of
|
||||
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
|
||||
corresponding invocation counts and budgets. This is an upper bound, not a
|
||||
promise that every call reaches a provider.
|
||||
|
||||
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
||||
effective generation timeout adds an inner request deadline, while zero
|
||||
disables only that generation deadline. The HTTP client timeout remains a
|
||||
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
||||
Repairs are serial within the same caller context, so their worst-case latency
|
||||
and cost follow the provider-call bound above; provision run deadlines and
|
||||
provider budgets accordingly. Credentials remain optional unless the selected
|
||||
PromptKit profile requires one, in which case preparation fails before a
|
||||
provider call when its configured credential is unavailable.
|
||||
The pinned upstream boundary and profile-format links are in
|
||||
[PromptKit Integration](integrations/pkg-promptkit.md).
|
||||
|
||||
@@ -279,6 +314,11 @@ positive value makes the effective active local-generation bound the smaller
|
||||
of **total_llm** and that local limit, so a local limit of four permits no more
|
||||
than four active local generations.
|
||||
|
||||
The Notarius scheduler admits one logical structured completion and holds that
|
||||
permit while PromptKit performs its serial corrective calls. PromptKit applies
|
||||
its selected-backend admission to each provider call; Notarius does not
|
||||
reacquire a permit or add another scheduler for a repair.
|
||||
|
||||
For a positive local limit, PromptKit owns its default waiting capacity and
|
||||
admission behavior. When a PromptKit backend has admitted all active and queued
|
||||
work, a new call fails as capacity exhaustion before generation. The adapter
|
||||
|
||||
@@ -195,6 +195,10 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
|
||||
and interpretation of structured output. Provider adapters do not own source-
|
||||
or domain-specific prompt logic.
|
||||
|
||||
PromptKit owns bounded structural correction within one structured completion.
|
||||
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
|
||||
the two budgets must remain separate.
|
||||
|
||||
When a model selects an application entity, callers must supply a contextual
|
||||
selection and deterministically attach the opaque application identity whenever
|
||||
the selection resolves exactly. Models do not receive or reproduce opaque
|
||||
@@ -261,6 +265,16 @@ contain application data and therefore inherits its sensitivity; operators own
|
||||
access controls and retention. Physical layout and operation are defined in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Platform And Distribution
|
||||
|
||||
Linux is the supported deployment platform. macOS is supported only as a
|
||||
best-effort development and compilation environment, while Windows is
|
||||
unsupported. Notarius distributes source releases only: an immutable source
|
||||
tag and its checked-in release note identify a release. The project does not
|
||||
publish executable binaries, archives, installers, container images,
|
||||
checksums, signatures, or package-manager entries. Maintainer release commands
|
||||
and tag guards belong to [Source Releases](../release.md).
|
||||
|
||||
## Architectural Non-Goals
|
||||
|
||||
Notarius does not aim to provide:
|
||||
|
||||
@@ -65,6 +65,8 @@ secret values.
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
|
||||
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
|
||||
crosses these topics, choose the document that owns the task and link to the
|
||||
other contracts.
|
||||
|
||||
### Releases
|
||||
|
||||
`docs/release.md` owns the source-release procedure. Release notes are
|
||||
historical summaries, not current-state contract owners: the checked-in note at
|
||||
an immutable tag records that release, while current canonical documentation
|
||||
must change with the behavior it describes. Do not use a release note to defer
|
||||
or replace current documentation updates.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
|
||||
164
docs/release.md
Normal file
164
docs/release.md
Normal file
@@ -0,0 +1,164 @@
|
||||
# Source Releases
|
||||
|
||||
This procedure is for maintainers publishing Notarius source releases. A
|
||||
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
|
||||
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
|
||||
candidate after publication; it does not publish or repair a release.
|
||||
|
||||
Notarius publishes no binaries, archives, checksums, signatures, containers,
|
||||
package-manager entries, or Gitea release objects. Windows is not supported.
|
||||
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
|
||||
`v0.3.0` tags.
|
||||
|
||||
## Select And Describe The Release
|
||||
|
||||
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
|
||||
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
|
||||
documented CLI, configuration, durable artifact, integration, or operating
|
||||
contract when its note explains the impact and required operator action. A
|
||||
patch release must not intentionally break those documented contracts within
|
||||
its minor line.
|
||||
|
||||
Create the version-matched note as part of the candidate. Every new note uses
|
||||
this structure, with concise, truthful content in each section:
|
||||
|
||||
```markdown
|
||||
# Notarius vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
The note is a historical summary. Link to current canonical documentation for
|
||||
exact behavior, and update that documentation in the candidate rather than
|
||||
using the note as a substitute.
|
||||
|
||||
## Prepare The Candidate
|
||||
|
||||
Set the selected release version and disable Go workspace use for every
|
||||
candidate command:
|
||||
|
||||
```sh
|
||||
RELEASE_VERSION=vMAJOR.MINOR.PATCH
|
||||
export RELEASE_VERSION GOWORK=off
|
||||
```
|
||||
|
||||
Run the shared source-candidate checks from the repository. They cover module
|
||||
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
|
||||
configuration validation, and the Linux and Darwin command-build matrix:
|
||||
|
||||
```sh
|
||||
./scripts/check-release-source.sh "$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
Before committing, manually follow every changed local Markdown link and
|
||||
review the candidate for unintended files, generated output, credentials, or
|
||||
other unrelated changes. Commit the release note and all affected current
|
||||
documentation, then run the shared checker against that exact candidate. Push
|
||||
the candidate commit to `main` only after it succeeds. Record the exact commit
|
||||
only after that push:
|
||||
|
||||
```sh
|
||||
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
```
|
||||
|
||||
For private-module installation, configure standard `GOPRIVATE` matching this
|
||||
module and ordinary Git authentication for the hosting service before running
|
||||
the verification below. The exact authentication mechanism belongs to the
|
||||
maintainer environment; never record credentials or environment dumps in a
|
||||
release note, command history, or repository file.
|
||||
|
||||
## Guard And Publish The Tag
|
||||
|
||||
Fetch current remote references, then run this guard without editing the
|
||||
candidate. It requires `main`, a clean worktree and index, disabled workspace
|
||||
use, a stable release version, the recorded and pushed commit, a matching note,
|
||||
and unused local and remote tags:
|
||||
|
||||
```sh
|
||||
git fetch origin main --tags
|
||||
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$GOWORK" = off
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
|
||||
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
|
||||
test -s "docs/releases/$RELEASE_VERSION.md"
|
||||
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
|
||||
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
|
||||
done
|
||||
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Create an explicitly lightweight tag against the guarded commit, verify its
|
||||
target, and push only that tag ref:
|
||||
|
||||
```sh
|
||||
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
|
||||
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
|
||||
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
|
||||
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
Never use `git push --tags`, move a published tag, or delete a published tag.
|
||||
|
||||
## Verify The Published Release
|
||||
|
||||
Confirm that the remote tag still points at the guarded commit and that the
|
||||
note is available from the tagged tree:
|
||||
|
||||
```sh
|
||||
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
|
||||
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
|
||||
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
|
||||
```
|
||||
|
||||
Verify a fresh source installation and its diagnostic version. The temporary
|
||||
directory confines the installed command to this check:
|
||||
|
||||
```sh
|
||||
release_verification_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
|
||||
mkdir -p "$release_verification_dir/bin"
|
||||
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
|
||||
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
|
||||
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
|
||||
```
|
||||
|
||||
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
|
||||
equivalent source verification when local installation policy requires it.
|
||||
`notarius --version` is diagnostic only; downstream compatibility remains
|
||||
defined by the published receipt and artifact contracts.
|
||||
|
||||
## Failure And Correction Policy
|
||||
|
||||
If candidate validation fails before publication, fix the candidate on `main`,
|
||||
rerun the shared checker, and repeat the guards. An unpublished local tag may
|
||||
be deleted after inspection.
|
||||
|
||||
If the remote tag or tag CI reveals a defect, leave the published tag intact.
|
||||
Fix the defect on `main`, choose a new patch version, write a new matching
|
||||
note, and repeat this procedure. Do not weaken tag immutability or add release
|
||||
assets as a workaround.
|
||||
224
docs/roadmap/dnd-subprocess-documentation.md
Normal file
224
docs/roadmap/dnd-subprocess-documentation.md
Normal file
@@ -0,0 +1,224 @@
|
||||
# D&D Subprocess Consumer Documentation
|
||||
|
||||
## Status
|
||||
|
||||
Completed. The target guide is `docs/consumers/dnd-pipeline.md`.
|
||||
|
||||
## Purpose
|
||||
|
||||
Provide one task-oriented guide for applications that run Notarius as a
|
||||
subprocess to execute the maintained complete D&D pipeline and consume its
|
||||
published artifacts. The initial concrete consumer is Narratio, but the guide
|
||||
must describe the public Notarius workflow rather than depend on Narratio
|
||||
internals.
|
||||
|
||||
The guide should make the safe integration path obvious without duplicating
|
||||
the CLI, input, receipt, output-bundle, or individual artifact contracts that
|
||||
already have canonical documentation.
|
||||
|
||||
## Current State
|
||||
|
||||
The public integration surface is documented accurately but is distributed
|
||||
across several documents:
|
||||
|
||||
- `docs/consumers/subprocess.md` defines the generic subprocess workflow;
|
||||
- `docs/cli.md` owns commands, flags, stream behavior, and exit statuses;
|
||||
- `docs/integrations/seriatim.md` owns the accepted transcript input shape;
|
||||
- `docs/integrations/run-result.md` owns the machine-readable successful-run
|
||||
receipt;
|
||||
- `docs/integrations/json-output.md` owns bundle discovery and logical files;
|
||||
- the D&D integration documents own the individual lane payload contracts;
|
||||
- `examples/dnd-complete.config.yml` is the maintained complete pipeline.
|
||||
|
||||
A consumer can reconstruct the full workflow from those documents, but there
|
||||
is no D&D-focused guide that connects the maintained example to its input,
|
||||
invocation, complete artifact inventory, discovery procedure, and downstream
|
||||
acceptance decisions.
|
||||
|
||||
## Target Documentation Set
|
||||
|
||||
### Create `docs/consumers/dnd-pipeline.md`
|
||||
|
||||
This document should own the end-to-end consumer workflow for the maintained
|
||||
complete D&D configuration. It should be useful to Narratio and to another
|
||||
subprocess orchestrator with the same needs.
|
||||
|
||||
The guide should contain the following sections.
|
||||
|
||||
#### Prerequisites And Deployment Configuration
|
||||
|
||||
- Link to `examples/dnd-complete.config.yml` rather than embedding a second
|
||||
complete configuration.
|
||||
- Explain that a deployment must provide the configured PromptKit profile and
|
||||
campaign reference files.
|
||||
- Recommend absolute paths for a service or orchestrator deployment.
|
||||
- Call out the path-resolution distinction explicitly: YAML reference paths
|
||||
are relative to the Notarius configuration file, while
|
||||
`promptkit.profile_file` is relative to the Notarius process working
|
||||
directory.
|
||||
- Recommend validating the selected configuration and `dnd-session` pipeline
|
||||
before processing sessions.
|
||||
|
||||
#### Transcript Input
|
||||
|
||||
- State that the complete pipeline consumes a Seriatim JSON document.
|
||||
- Link to the canonical Seriatim contract for required fields and validation.
|
||||
- Recommend the caller's final trimmed transcript when the caller maintains
|
||||
transcript tiers. For Narratio, identify the implemented source as
|
||||
`narratio.transcript.final_trimmed`, normally stored at
|
||||
`transcripts/final.trimmed.json`.
|
||||
- Explain that segment IDs must remain stable because D&D source references
|
||||
cite those units.
|
||||
- Explain that Notarius derives its default prompt session from the input
|
||||
module and exact input bytes and that ordinary callers should not supply
|
||||
`--session-id`.
|
||||
|
||||
#### Subprocess Invocation
|
||||
|
||||
- Show one concise invocation using `notarius run dnd-session`, explicit
|
||||
absolute `--config`, `--input`, and `--output-dir` paths, and `--json`.
|
||||
- Direct callers to capture stdout and stderr separately, propagate
|
||||
cancellation, impose an operator-appropriate timeout, and wait for process
|
||||
completion before parsing stdout.
|
||||
- State that only exit status zero permits receipt decoding and link to the CLI
|
||||
contract for the complete exit-status definition.
|
||||
- Recommend retaining stderr and the invocation context for diagnosis without
|
||||
logging secrets or transcript content.
|
||||
|
||||
#### Receipt And Bundle Discovery
|
||||
|
||||
- Require callers to accept only supported run-result schema versions while
|
||||
tolerating unknown fields allowed by that version.
|
||||
- Direct callers to obtain the exact run-specific bundle from the receipt's
|
||||
absolute `output_directory`; they must not scan for the newest run directory
|
||||
or construct a run ID.
|
||||
- Require a confinement check when resolving `index_file` beneath the reported
|
||||
bundle root.
|
||||
- Direct callers to discover lane payloads by `lane_id` in `index.json`, then
|
||||
verify descriptor media type and schema identity before decoding them.
|
||||
- Explain that descriptor paths are untrusted relative paths and require the
|
||||
same confinement discipline.
|
||||
|
||||
#### Complete D&D Artifact Inventory
|
||||
|
||||
Include a compact table for the ten lane IDs selected by the maintained
|
||||
complete configuration:
|
||||
|
||||
- `item-registry`;
|
||||
- `npc-registry`;
|
||||
- `location-registry`;
|
||||
- `scene-descriptions`;
|
||||
- `item-occurrences`;
|
||||
- `spells`;
|
||||
- `combat-turns`;
|
||||
- `npc-occurrences`;
|
||||
- `location-occurrences`;
|
||||
- `enemy-events`.
|
||||
|
||||
For each row, give a one-line purpose and link to the corresponding canonical
|
||||
D&D artifact contract. Do not copy its fields or schema rules into the
|
||||
consumer guide.
|
||||
|
||||
Document the four always-published bundle files—`index.json`, `manifest.json`,
|
||||
`rejected.json`, and `warnings.json`—and the complete example's configured
|
||||
`chunk-map.json` and `evidence-context.json` pipeline-wide artifacts. Link to
|
||||
their canonical contracts and distinguish pipeline-wide artifacts from lane
|
||||
outputs.
|
||||
|
||||
The inventory must say that a file is available only when its corresponding
|
||||
artifact was accepted and published. It must not imply that process success
|
||||
guarantees every configured lane.
|
||||
|
||||
#### Downstream Acceptance And Retention
|
||||
|
||||
- Explain that exit status zero can coexist with rejected outputs, warnings,
|
||||
or absent lane descriptors.
|
||||
- Require the consumer to define its required lane set explicitly. Recommend
|
||||
treating all ten lanes as required when the caller claims to consume the
|
||||
complete D&D workflow, while allowing another consumer to adopt a narrower
|
||||
documented policy.
|
||||
- Recommend retaining the receipt, the complete published bundle, and captured
|
||||
diagnostic streams long enough to support provenance and failure analysis.
|
||||
- Explain that `evidence-context.json` is a reading excerpt; authoritative
|
||||
citations remain in lane payloads.
|
||||
- Treat transcripts, lane artifacts, evidence context, manifests, and logs as
|
||||
sensitive campaign data.
|
||||
|
||||
#### Compatibility Checklist
|
||||
|
||||
End with a concise checklist covering process exit, receipt schema, path
|
||||
confinement, pipeline identity, index decoding, required descriptors,
|
||||
descriptor schema/media compatibility, warnings and rejections, checksums or
|
||||
retention, and secure handling. Compatibility should be based on published
|
||||
receipt and artifact contracts rather than parsing a human version string.
|
||||
|
||||
### Update Existing Navigation
|
||||
|
||||
- Add a short link from `docs/consumers/subprocess.md` to the D&D-specific
|
||||
workflow. Keep generic subprocess policy in the existing document.
|
||||
- Add the guide to the documentation links in `README.md`.
|
||||
- Extend the subprocess-consumer row in `docs/development.md` so maintainers
|
||||
working on the D&D workflow are routed to the new guide and the canonical
|
||||
contracts.
|
||||
|
||||
### Verify Canonical Contract Documents
|
||||
|
||||
Review the linked integration documents and the complete example while writing
|
||||
the guide. Correct an integration document only if repository inspection finds
|
||||
an actual stale contract. Do not move schema definitions, field tables, CLI
|
||||
flags, or configuration semantics into the new guide.
|
||||
|
||||
## Narratio Alignment
|
||||
|
||||
The guide may name Narratio as the motivating consumer and identify its current
|
||||
final-trimmed transcript source. It must not claim that Narratio already has a
|
||||
Notarius adapter or extraction stage. Until that feature is implemented,
|
||||
Narratio-specific architecture, configuration, stage behavior, manifest
|
||||
records, and artifact source IDs belong in Narratio's roadmap.
|
||||
|
||||
Once Narratio implements the integration, its own integration documentation
|
||||
should link to this guide and the durable Notarius contracts instead of
|
||||
repeating them.
|
||||
|
||||
## Validation
|
||||
|
||||
Documentation implementation should include:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Also verify all new and changed relative Markdown links, compare the artifact
|
||||
inventory directly with the maintained complete configuration, and confirm
|
||||
that commands and path semantics match the CLI and configuration references.
|
||||
If the repository still has no automated link checker, record that fact and
|
||||
perform a focused manual link review.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A subprocess integrator can follow one D&D-focused guide from a Seriatim
|
||||
transcript through safe discovery of every artifact configured by the
|
||||
complete example.
|
||||
- The guide makes stdout, stderr, exit-status, receipt, and path-confinement
|
||||
responsibilities unambiguous.
|
||||
- The ten configured D&D lanes and both configured pipeline-wide artifacts are
|
||||
listed and linked to their canonical contracts.
|
||||
- The guide distinguishes process success from the caller's required-artifact
|
||||
policy.
|
||||
- The profile-path and reference-path resolution rules are clearly stated.
|
||||
- Existing navigation makes the guide discoverable.
|
||||
- No volatile contract is defined in two places, and no unimplemented Narratio
|
||||
behavior is presented as current.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Implementing or documenting Narratio's future adapter or stage as current
|
||||
Notarius behavior.
|
||||
- Adding a new Notarius command, receipt version, output format, or artifact
|
||||
schema.
|
||||
- Duplicating the complete configuration or individual D&D payload schemas in
|
||||
prose.
|
||||
- Defining a universal partial-result policy for every Notarius consumer.
|
||||
@@ -5,13 +5,207 @@ configuration, operations, internal, and integration docs. This roadmap records
|
||||
future work only. Items are ordered roughly by current value and specificity,
|
||||
not as committed release dates.
|
||||
|
||||
## Near-Term Validation And LLM Reliability
|
||||
|
||||
The following work forms one related program but should be promoted into
|
||||
separate feature roadmaps and implemented in dependency order. PromptKit owns
|
||||
structural output repair within one completion. Notarius owns stage candidates,
|
||||
validator chains, semantic rejection policy, and whether another stage attempt
|
||||
is warranted.
|
||||
|
||||
### 1. Upgrade To PromptKit v0.8.0
|
||||
|
||||
This item has been promoted to the standalone
|
||||
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md) roadmap. That document owns the
|
||||
release-by-release compatibility review, adopted features, structured-repair
|
||||
policy, target integration boundary, acceptance criteria, and settled design
|
||||
decisions.
|
||||
|
||||
### 2. Feedback-Aware Stage Validation Retries
|
||||
|
||||
- Model Notarius's corrective stage-retry conversation explicitly after
|
||||
PromptKit v0.8.0. The first attempt sends the ordinary complete initial
|
||||
prompt. If application validation rejects the resulting LLM-produced
|
||||
candidate and another stage attempt is available, reconstruct that complete
|
||||
initial prompt byte-for-byte and append exactly two messages: an assistant
|
||||
message containing the defective response and an application-owned user
|
||||
message detailing every applicable semantic validation error and requesting
|
||||
one corrected, complete replacement response. This is a freshly constructed
|
||||
correction request, not continuation of an accumulating conversation.
|
||||
- Use the configured stage `retries` value as the one outer retry budget for
|
||||
this loop. `retries: N` continues to mean at most `N` additional complete
|
||||
chunk, extract, merge, or normalize attempts after the initial attempt,
|
||||
whether an attempt is needed because of a producer error or semantic
|
||||
rejection. Do not add a second semantic-correction count. PromptKit's
|
||||
prompt-level `repair_attempts` budget is independent and internal to each
|
||||
individual LLM completion, and does not consume or replenish the Notarius
|
||||
stage budget.
|
||||
- Extend the framework-managed validation boundary for chunk, extract, merge,
|
||||
and normalize stages so a rejected LLM-produced candidate and its exact raw
|
||||
model response remain available to construct the next stage attempt.
|
||||
Deterministic producers cannot improve by repeating the same inputs; a
|
||||
rejection from a deterministic stage is therefore terminal under the
|
||||
configured rejection policy rather than consuming retries mechanically.
|
||||
- Preserve the original session ID, selected profile, structured-output
|
||||
contract, prompt inputs, and reusable prompt prefix. Carry only the latest
|
||||
candidate and latest aggregate feedback; do not build an unbounded retry
|
||||
conversation. Keep model-facing corrective guidance separate from
|
||||
operator-facing diagnostics, and apply explicit size, redaction, and debug
|
||||
disclosure rules to both.
|
||||
- Run every applicable validator in the configured chain before deciding
|
||||
whether to retry. Do not short-circuit merely because an earlier validator
|
||||
rejected the candidate. Aggregate all semantic rejection reason codes and
|
||||
corrective guidance into the retry message so one retry can address the
|
||||
whole candidate. A validator is applicable only when its declared target and
|
||||
prerequisites can be satisfied; record a deterministic skipped diagnostic
|
||||
rather than invoking a validator on an input it cannot interpret. Initially
|
||||
execute the chain sequentially in configured order so results, diagnostics,
|
||||
costs, and feedback ordering remain deterministic; consider validator
|
||||
concurrency only in response to measured latency.
|
||||
- Continue running independent applicable validators after one validator
|
||||
execution failure so the attempt retains as much useful diagnostic
|
||||
information as practical. Do not present validator operational failures as
|
||||
defects in the producer candidate and do not include them in corrective
|
||||
feedback.
|
||||
- Distinguish three terminal conditions and make their policies configurable
|
||||
at a coherent pipeline or binding scope:
|
||||
- **producer structural failure:** PromptKit could not return a usable
|
||||
structured candidate after its repair budget. Default to `fail_run`; an
|
||||
allowed alternative may record a terminal stage or lane rejection where
|
||||
execution can safely continue, but may not accept the invalid output;
|
||||
- **semantic rejection:** one or more validators completed and rejected the
|
||||
candidate. Default to `fail_run` after corrective stage retries are
|
||||
exhausted; allow an explicit alternative that records the existing
|
||||
rejected-output outcome without advancing that output;
|
||||
- **validator execution failure:** a validator could not produce a valid
|
||||
decision because of generation, structural-output, transport, or internal
|
||||
failure. Default to a genuine warning and an explicitly recorded
|
||||
`validation_incomplete` or equivalent degraded state while allowing the
|
||||
candidate to continue; allow strict configuration to fail the run instead.
|
||||
- An LLM-backed validator uses the same scheduled PromptKit boundary as every
|
||||
other LLM-backed module. Its own response may use PromptKit's bounded
|
||||
structural repair. Distinguish its possible output states:
|
||||
- output rejected by PromptKit's structural contract should consume only the
|
||||
validator prompt's configured PromptKit repair budget;
|
||||
- output that is structurally valid but violates a deterministically
|
||||
checkable validator-result invariant should be classified as a validator
|
||||
execution failure;
|
||||
- output that satisfies the complete validator-result contract is the
|
||||
validator's decision, even though an LLM judgment may remain imperfect.
|
||||
Automatically judging that judgment would require another semantic
|
||||
validator and is outside this feature.
|
||||
If the validator cannot return a contract-valid decision, do not recursively
|
||||
create another Notarius semantic-validation loop around it. Apply the
|
||||
configured validator-failure policy. The default warning must identify the
|
||||
validator and affected stage without exposing sensitive content.
|
||||
- Separate validator execution retry from producer correction. A transient
|
||||
validator operational failure must not automatically discard and regenerate
|
||||
an otherwise usable producer candidate. Any bounded retry of the validator
|
||||
itself should reuse that same immutable candidate and remain subordinate to
|
||||
PromptKit and provider retry behavior.
|
||||
- Preserve attempt-level provenance, cumulative token usage, validator
|
||||
outcomes, aggregated correction feedback, and terminal policy decisions in
|
||||
the debug and manifest models without copying raw source material into
|
||||
ordinary errors or durable summaries.
|
||||
- Define terminal-outcome precedence. A semantic rejection dominates a
|
||||
validator execution failure for the same candidate: use the completed
|
||||
rejections to correct the producer while separately recording incomplete
|
||||
validation. If a later candidate has no semantic rejection but one validator
|
||||
still fails, apply the configured validator-failure policy to that candidate.
|
||||
Never allow a known semantic rejection to become accepted through a
|
||||
warn-and-continue setting, and never accept a structurally invalid producer
|
||||
response. Permissive policy may preserve a rejected-output outcome or accept
|
||||
a structurally valid candidate with explicitly incomplete validation; it may
|
||||
not relabel known-invalid output as approved.
|
||||
|
||||
Before implementation, record the generic validation and retry state machine
|
||||
in an ADR. The ADR should own the separation between PromptKit repair and
|
||||
Notarius correction, use of the existing stage-retry budget, reconstruction of
|
||||
correction conversations, all-applicable-validator aggregation, deterministic
|
||||
validator ordering, non-recursive validator failure handling, outcome
|
||||
precedence, default fail-open/fail-closed choices, configurable terminal
|
||||
policies, and provenance and sensitive-data constraints. A dependency-upgrade
|
||||
ADR is not needed for PromptKit v0.8.0 itself. Current behavior remains
|
||||
authoritative until the validation ADR is implemented and the canonical
|
||||
architecture, configuration, operations, and internal documentation are
|
||||
updated.
|
||||
|
||||
### 3. D&D Combat Scene Semantic Validation
|
||||
|
||||
- Add an optional production LLM-backed D&D validator that determines whether
|
||||
proposed scene boundaries and classifications represent substantive active
|
||||
combat correctly. Its central quality goal is that active combat is kept in
|
||||
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||
- Resolve the validator's exact target before implementation. The current
|
||||
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||
`recap`, and `meta` classification. The preferred initial placement is
|
||||
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||
can compare one proposed kind with the corresponding transcript chunk.
|
||||
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||
question that can be answered from the complete transcript and proposed
|
||||
range map, such as whether one continuous combat was fragmented across
|
||||
inappropriate scene boundaries. Do not duplicate the same classification
|
||||
judgment at both stages. Moving classification into chunk-plan annotations
|
||||
would change the deliberately minimal, annotation-free chunk contract and
|
||||
requires an explicit architecture review before it is selected.
|
||||
- Validate both false negatives and false positives: a non-combat kind must not
|
||||
omit substantive active combat, and a combat kind must be supported by such
|
||||
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||
extraction runs only for an exact `combat` scene classification; semantic
|
||||
review improves the upstream classification but does not replace that gate.
|
||||
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||
structured response schema, and let PromptKit repair structural validator
|
||||
output within its bounded budget. A contract-invalid final validator response
|
||||
is a validator execution failure, not a semantic rejection and not a reason
|
||||
to recursively validate the validator.
|
||||
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||
added calls, latency, and token cost before placing it in the production
|
||||
default chain.
|
||||
- An ADR is not required if classification remains owned by
|
||||
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||
Create or supersede an ADR if the work transfers scene classification into
|
||||
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||
contract.
|
||||
|
||||
### 4. Warning Signal And Presentation Reform
|
||||
|
||||
- Audit every warning producer and representative successful runs. Ordinary
|
||||
success producing dozens of warnings is a failed operator experience: the
|
||||
volume obscures actionable problems and trains operators to ignore the
|
||||
warning channel.
|
||||
- Define a small warning taxonomy that distinguishes actionable degradation,
|
||||
incomplete validation, lossy fallback, and data-quality risk from routine
|
||||
normalization observations or informational diagnostics. Preserve detailed
|
||||
traceability in debug or manifest data without promoting every observation
|
||||
to a top-level CLI warning.
|
||||
- Consider stable deduplication and aggregation by scope and reason code,
|
||||
bounded samples plus omitted counts, and a concise CLI summary with a path to
|
||||
detailed diagnostics. Do not suppress genuine validator execution failures
|
||||
merely to reduce the count.
|
||||
- Decide which warnings affect process status, rejection summaries, durable run
|
||||
receipts, or only debug output. Ensure warning ordering and aggregation are
|
||||
deterministic across concurrent execution.
|
||||
- Establish a representative warning-volume acceptance target and human review
|
||||
workflow before changing individual producers piecemeal. The intended result
|
||||
is not zero warnings; it is a small set in which every surfaced warning merits
|
||||
operator attention.
|
||||
- This work does not require an ADR unless it changes validation acceptance,
|
||||
failure, or durable contract semantics. CLI presentation and diagnostic
|
||||
taxonomy otherwise belong in a feature roadmap followed by updates to their
|
||||
canonical configuration, operations, integration, and internal documents.
|
||||
|
||||
## Near-Term D&D Pipeline
|
||||
|
||||
### Evaluate Spell Extraction And Normalization
|
||||
|
||||
- Evaluate ordinary extraction retries and the completed normalization path
|
||||
against a human-reviewed transcript set before adding repair-aware retries or
|
||||
an LLM-backed semantic validator.
|
||||
against a human-reviewed transcript set before and after adopting the shared
|
||||
PromptKit repair and Notarius validation-retry policies above.
|
||||
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
||||
validator, and normalizer development. Treat model-quality review as an
|
||||
iterative human evaluation aid, not a deterministic correctness gate.
|
||||
@@ -28,8 +222,7 @@ The implemented source-backed core and initial D&D registry adoption are
|
||||
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
|
||||
and
|
||||
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||
The [Semantic Reconciliation Roadmap](semantic-reconciliation.md) retains the
|
||||
original feature scope; the sections below keep broader extensions deferred.
|
||||
The sections below keep broader extensions deferred.
|
||||
|
||||
### Large-Collection Semantic Reconciliation
|
||||
|
||||
@@ -153,7 +346,6 @@ section only after a concrete workflow, contract, and priority emerge.
|
||||
### Distribution And Operations
|
||||
|
||||
- Packaged release artifacts for alpha distribution.
|
||||
- A documented versioning and release process.
|
||||
- Optional generated example-output fixtures with a regeneration procedure.
|
||||
- Additional diagnostics or reporting views.
|
||||
|
||||
|
||||
782
docs/roadmap/implementation.md
Normal file
782
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,782 @@
|
||||
# PromptKit v0.8.0 Upgrade Implementation Plan
|
||||
|
||||
## Purpose
|
||||
|
||||
Implement the target state defined by the
|
||||
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md): adopt the useful PromptKit
|
||||
v0.6.0, v0.7.0, and v0.8.0 changes; enable one bounded structural correction
|
||||
by default; expose pipeline and binding overrides; preserve safe provider
|
||||
diagnostics; and keep PromptKit behind Notarius's transport-neutral LLM
|
||||
boundary.
|
||||
|
||||
This plan is ordered. Each numbered stage is one implementation prompt for a
|
||||
gpt-5.6-terra coding agent. Complete and validate one stage before beginning
|
||||
the next. Read `docs/development.md` and every policy under `docs/policy/` at
|
||||
the start of each stage, inspect the current code and tests named by that
|
||||
stage, preserve unrelated worktree changes, and update current-behavior
|
||||
documentation in the same stage as the behavior it describes.
|
||||
|
||||
Do not retire this plan or `promptkit-v0.8.md` during implementation. Keep both
|
||||
until the completed work has passed a separate review. Do not implement the
|
||||
future Notarius semantic-validation retry loop, D&D combat-scene validator, or
|
||||
warning redesign as part of this plan.
|
||||
|
||||
## Decisions Fixed For Implementation
|
||||
|
||||
- Pin `gitea.maximumdirect.net/eric/promptkit` v0.8.0 directly, with no
|
||||
`replace`, workspace dependency, or vendored source.
|
||||
- Every maintained eligible production prompt defaults to exactly one
|
||||
PromptKit structural repair attempt.
|
||||
- Add the exact configuration key
|
||||
`structured_output_repair_attempts` at pipeline scope and on LLM-backed
|
||||
module and validator bindings.
|
||||
- Effective precedence is binding value, then pipeline value, then the prompt's
|
||||
declared `repair_attempts` value. Omission inherits; explicit zero disables
|
||||
structural repair at that scope.
|
||||
- Accepted values are integers from zero through three. Explicit null and
|
||||
non-integer values are invalid. An explicit binding value on a deterministic
|
||||
module or validator is invalid. A pipeline value is applied only to selected
|
||||
LLM-backed bindings and does not make deterministic bindings invalid.
|
||||
- Keep file configuration version 4. This is an additive pre-release field and
|
||||
does not require parallel versioned behavior.
|
||||
- Use `StructuredOutputRepairAttempts *int` for presence-aware internal Go
|
||||
fields. Clone pointers at every ownership boundary.
|
||||
- A configured override never replaces schema identity, output format, or
|
||||
validation mode. The PromptKit adapter calls `InspectPrompt`, copies the
|
||||
complete normalized prompt-owned output contract, changes only
|
||||
`RepairAttempts`, and supplies the complete replacement on `RunRequest`.
|
||||
Do not add an inspection cache initially.
|
||||
- PromptKit repair is internal to one `CompleteStructured` call and does not
|
||||
consume or replenish a binding's existing `retries` budget.
|
||||
- Add `RepairAttempts int` to Notarius's structured-completion response. It is
|
||||
the actual corrective-call count reported by PromptKit; token usage remains
|
||||
PromptKit's cumulative usage and must not be summed again.
|
||||
- A valid repaired response is successful and produces no warning solely
|
||||
because repair occurred. Exhausted structural validation maps to
|
||||
`ErrInvalidStructuredOutput` with the final candidate and debug material
|
||||
retained.
|
||||
- Add an application-owned generation-error sentinel and typed status-bearing
|
||||
error. PromptKit error types must not cross `internal/framework/llm`.
|
||||
- HTTP status may appear in the application-owned generation error. Provider
|
||||
code, type, and message are excluded from ordinary errors, warnings,
|
||||
manifests, cache, and checkpoint identity; they may appear only in an
|
||||
explicitly requested debug trace after Notarius redaction.
|
||||
- Profile inheritance is owned entirely by PromptKit. Notarius passes sources
|
||||
through, inspects and records the resolved target, and does not parse or merge
|
||||
`base_profile` itself.
|
||||
- PromptKit's built-in `rakestrawhome` backend and
|
||||
`rakestrawhome-gemma-4-31b` profile are available generically. Notarius does
|
||||
not register, shadow, or select them by default.
|
||||
- Missing optional credential environment values are allowed to reach the
|
||||
provider without `Authorization`; Notarius does not recreate v0.5.0's local
|
||||
failure or add provider-specific authentication logic.
|
||||
- No dependency-upgrade ADR is required. Update architecture only with the
|
||||
durable ownership distinction between PromptKit structural repair and
|
||||
Notarius stage/semantic validation policy.
|
||||
|
||||
## Stage 1: Upgrade The Dependency And Establish A Clean v0.8.0 Baseline ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Move the repository to PromptKit v0.8.0, resolve source-compatibility issues,
|
||||
and establish a passing baseline before adopting new behavior.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Re-read the upstream v0.6.0, v0.7.0, and v0.8.0 release guides and the
|
||||
v0.8.0 package consumer and format documentation. Treat the pinned v0.8.0
|
||||
tag, not the sibling checkout's moving branch, as authoritative.
|
||||
2. Update `go.mod` and `go.sum` to PromptKit v0.8.0 and run `go mod tidy` with
|
||||
`GOWORK=off`.
|
||||
3. Compile before making compatibility edits. Correct only actual source or
|
||||
behavior incompatibilities. In particular:
|
||||
- convert any positional `promptkit.Profile` or
|
||||
`promptkit.OpenAICompatibleProfileConfig` literals to keyed literals;
|
||||
- confirm Notarius does not register the newly reserved `rakestrawhome`
|
||||
backend ID; and
|
||||
- preserve `PrepareExecution`/`Details`/`RunPrepared` snapshot ownership,
|
||||
`Discard`, session forwarding, reasoning override, profile preflight,
|
||||
and capacity adaptation.
|
||||
4. Change `promptKitBuiltinProfileCatalogID` in
|
||||
`internal/framework/llm/promptkit_profile_fingerprint.go` from the v0.5.0
|
||||
catalog marker to an opaque v0.8.0 marker. Do not hash PromptKit internal
|
||||
files or include catalog content in manifests.
|
||||
5. Update `docs/integrations/pkg-promptkit.md` to pin and link v0.8.0 and to
|
||||
state that this stage still leaves the production prompt-declared repair
|
||||
budget at its current value. Do not document later configuration or default
|
||||
behavior before it exists.
|
||||
6. Update only those existing tests whose public PromptKit types or stable
|
||||
v0.8.0 behavior genuinely changed. Do not rewrite tests merely to match
|
||||
upstream diagnostic wording.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go mod tidy -diff
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./cmd/notarius
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- `go list -m gitea.maximumdirect.net/eric/promptkit` reports v0.8.0.
|
||||
- There is no PromptKit `replace`, active Go workspace dependency, or vendor
|
||||
tree.
|
||||
- The adapter still uses one frozen prepared execution and all existing LLM
|
||||
tests pass.
|
||||
- Checkpoint profile identity includes the v0.8.0 built-in catalog marker.
|
||||
- Current integration documentation pins v0.8.0 without claiming that
|
||||
later stages are already active.
|
||||
- The full ordinary test suite, vet, and command build pass.
|
||||
|
||||
## Stage 2: Verify v0.6.0 Compatibility And Hardening ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Audit Notarius's assets and boundary values against PromptKit v0.6.0's stricter
|
||||
source, path, endpoint, JSON, and cancellation contracts, fixing only concrete
|
||||
incompatibilities.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inspect `internal/framework/llm/asset_registry.go`, prompt/profile source
|
||||
composition, all registered asset roots, the conventional local backend,
|
||||
and their focused tests.
|
||||
2. Exercise every production asset registry through PromptKit engine
|
||||
construction and the existing production composition tests. Confirm that:
|
||||
- YAML IDs and versions, not filenames, select definitions;
|
||||
- every `content_file` path is exact, relative, contained, and points to a
|
||||
regular embedded file;
|
||||
- every schema and JSON asset is one complete JSON value;
|
||||
- every current output contract is valid under v0.8.0; and
|
||||
- unrelated malformed definitions do not create a second Notarius identity
|
||||
or fallback mechanism.
|
||||
3. Review local endpoint parsing and validation. Retain a narrower Notarius
|
||||
rule only if it has independent application value; otherwise rely on
|
||||
PromptKit's absolute HTTP/HTTPS URL contract. Never accept a value that the
|
||||
adapter will later reject.
|
||||
4. Review conversion of Notarius variables, inputs, profile extras, and debug
|
||||
values at the adapter boundary for PromptKit's bounded JSON-compatible-value
|
||||
rules. Do not add a second generic JSON walker or duplicate upstream numeric
|
||||
limits.
|
||||
5. Verify cancellation and deadline identity through existing adapter tests.
|
||||
Add or refine one focused regression only if Notarius currently destroys an
|
||||
`errors.Is`-relevant context or transport error that the application owns.
|
||||
6. Do not add a cross-operation schema cache, artifact cache, provider-body
|
||||
reader, or duplicate JSON framing validation; v0.6.0 owns those mechanisms.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run the focused asset, profile-source, and adapter packages, then the ordinary
|
||||
and race-enabled suites:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Tests must remain offline and should validate Notarius's assembled boundary,
|
||||
not reproduce PromptKit's internal path, JSON-depth, or response-size matrices.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every maintained embedded prompt, schema, and fallback profile can be loaded
|
||||
through the assembled v0.8.0 engine.
|
||||
- Current local endpoint and JSON-compatible values either satisfy the stricter
|
||||
upstream contract or fail during preparation with safe diagnostics.
|
||||
- No duplicate PromptKit-owned cache, JSON, or response-bound mechanism is
|
||||
introduced.
|
||||
- Cancellation and deadline behavior remains discoverable at the Notarius
|
||||
boundary.
|
||||
- Ordinary and race-enabled tests pass.
|
||||
|
||||
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Make the useful PromptKit v0.7.0 profile and backend behavior work through
|
||||
Notarius's existing generic profile boundary without adding provider-specific
|
||||
composition logic.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inspect `promptkit_profiles.go`, `asset_registry.go`, profile fingerprinting,
|
||||
CLI profile preflight, profile provenance recording, and their tests before
|
||||
editing.
|
||||
2. Add an offline integration test using a temporary operator profile source
|
||||
whose leaf uses `base_profile`. Prove that:
|
||||
- preflight reports the leaf ID;
|
||||
- the effective backend, model, reasoning, and other inherited values match
|
||||
the resolved PromptKit target;
|
||||
- execution uses the same resolved target as inspection; and
|
||||
- a missing parent or cycle fails before provider generation with a safe
|
||||
profile-load diagnostic.
|
||||
Do not duplicate PromptKit's entire field-by-field merge test matrix.
|
||||
3. Add a checkpoint-safety test showing that changing a parent definition in
|
||||
an operator profile directory changes Notarius's profile-source fingerprint
|
||||
while profile content and paths remain absent from the fingerprint value.
|
||||
Retain the v0.8.0 catalog marker as coverage for built-in-parent changes.
|
||||
4. Verify `rakestrawhome-gemma-4-31b` through the ordinary profile inspector.
|
||||
Assert its selected backend reaches Notarius's application-owned inspection
|
||||
and provenance fields. Use a fake PromptKit client or transport if execution
|
||||
coverage is needed; never contact the live service or require credentials.
|
||||
5. Verify that Notarius registers no `rakestrawhome` override and that the
|
||||
existing `local` registration remains independent.
|
||||
6. Add one `httptest`-backed adapter integration test for a filesystem profile
|
||||
with a missing optional `api_key_env`. The request must reach the test server
|
||||
without an `Authorization` header. Add a focused in-memory PromptKit profile
|
||||
test for `APIKeyRequired` only if needed to prove Notarius preserves upstream
|
||||
preflight behavior; do not expose a new operator profile API.
|
||||
7. Keep `assets/dnd/profiles/dnd-extraction.yaml` standalone and unchanged. No
|
||||
matching v0.8.0 built-in profile owns its `openai/gpt-5.6-luna` target.
|
||||
8. Update the current profile-source, deployment, and pinned-integration
|
||||
sections in `docs/config.md`, `docs/operations.md`,
|
||||
`docs/internal/llm.md`, and `docs/integrations/pkg-promptkit.md`. Link to the
|
||||
pinned PromptKit format rules for inheritance. Explain that filesystem
|
||||
profiles cannot express PromptKit's in-memory `APIKeyRequired` field and
|
||||
that an optional missing credential may result in a provider 401/403.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/core/config ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/cli
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Inherited operator profiles resolve identically during preflight and
|
||||
execution, with the leaf ID and effective target kept distinct.
|
||||
- Parent changes invalidate checkpoint reuse without leaking profile content or
|
||||
paths.
|
||||
- Rakestrawhome is available through generic PromptKit profile handling and is
|
||||
not selected by default or registered by Notarius.
|
||||
- Missing optional credentials omit authorization and reach the controlled
|
||||
test provider; explicitly required credentials retain upstream behavior.
|
||||
- Current documentation accurately describes the implemented profile and
|
||||
credential behavior without duplicating PromptKit's merge algorithm.
|
||||
|
||||
## Stage 4: Adapt Structured Generation Errors Safely ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Use PromptKit v0.7.0's structured generation errors for stable status
|
||||
classification and debug-only provider diagnostics without leaking PromptKit
|
||||
types or sensitive provider text.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. In `internal/framework/contracts`, add:
|
||||
- `ErrLLMGeneration` as the provider-neutral generation-failure sentinel;
|
||||
- an application-owned `LLMGenerationError` with private status and safe
|
||||
diagnostic fields, `Error`, `Unwrap`, and `StatusCode` methods; and
|
||||
- a constructor that accepts a nonnegative status and an already-redacted
|
||||
diagnostic. Status zero means no HTTP status was available.
|
||||
Ordinary callers may inspect status with `errors.As` and category with
|
||||
`errors.Is`, but cannot obtain provider code, type, or message from the
|
||||
error.
|
||||
2. Add an application-owned `LLMDebugProviderError` with `status_code`, `code`,
|
||||
`type`, and `message` fields, referenced optionally from
|
||||
`LLMDebugResponse`. This is debug material, not a manifest or durable public
|
||||
artifact contract.
|
||||
3. In `PromptKitClient.CompleteStructured`, preserve precedence in this order:
|
||||
caller context cancellation/deadline, PromptKit capacity error, structured
|
||||
PromptKit generation error, then other PromptKit generation failures.
|
||||
Map every generation failure to `ErrLLMGeneration`; map
|
||||
`*promptkit.GenerationError` to `LLMGenerationError` with its status.
|
||||
Never wrap or return the PromptKit error value itself.
|
||||
4. Keep the ordinary diagnostic limited to PromptKit's safe default error
|
||||
formatting after bearer and known-credential redaction. Do not append
|
||||
`ProviderCode`, `ProviderType`, or `ProviderMessage` to it.
|
||||
5. For an explicitly requested debug path, preserve prepared prompt details and
|
||||
attach the PromptKit provider code, type, and message after:
|
||||
- reading only the selected prepared target's `APIKeyEnv`, if any, to obtain
|
||||
the exact known credential solely for redaction;
|
||||
- applying `RedactSecrets` and the existing bearer/key-pattern redaction;
|
||||
- retaining PromptKit's already-normalized bounds; and
|
||||
- discarding the credential value immediately rather than storing it.
|
||||
Do not scan unrelated environment variables.
|
||||
6. Return prompt/debug material alongside the error so the existing debug LLM
|
||||
wrapper can persist it only when debug recording is enabled. Confirm that
|
||||
provider fields do not appear in ordinary error text, warnings, manifests,
|
||||
cache, checkpoint data, or a run without debug output.
|
||||
7. Refactor error mapping into small helpers if needed to keep
|
||||
`CompleteStructured` readable; do not create provider-specific policy in
|
||||
modules or the pipeline runner.
|
||||
8. Update the error and observability sections of `docs/internal/llm.md` and
|
||||
`docs/integrations/pkg-promptkit.md`. Keep operator disclosure rules in
|
||||
`docs/operations.md` concise and link to the internal boundary where useful.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
- Use `httptest.Server` to return representative structured 400 and 503
|
||||
responses. Assert `errors.Is(ErrLLMGeneration)`, `errors.As` to the
|
||||
application-owned type, and the exact status without asserting complete
|
||||
human wording.
|
||||
- Include a provider message containing the selected test credential and a
|
||||
bearer-shaped value. Verify both are absent from the ordinary error and
|
||||
debug artifact, while a non-sensitive marker appears only in the requested
|
||||
debug trace.
|
||||
- Retain existing capacity and context tests to prove their more specific
|
||||
classifications still win.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- PromptKit generation errors never escape the adapter error chain.
|
||||
- All generation failures match `ErrLLMGeneration`; structured non-success
|
||||
responses expose only application-owned HTTP status to ordinary callers.
|
||||
- Provider code, type, and message are available only in an explicitly
|
||||
requested, redacted debug trace.
|
||||
- Capacity and context classifications remain unchanged and more specific.
|
||||
- Security tests prove selected credentials and bearer tokens are not leaked.
|
||||
|
||||
## Stage 5: Add Adapter-Level Structured Repair Support ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Teach the transport-neutral completion boundary and PromptKit adapter to apply
|
||||
an optional repair override and report actual repair behavior, without yet
|
||||
exposing the setting in pipeline configuration.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` to
|
||||
`contracts.StructuredCompletionRequest`. Copy the pointed-to value wherever
|
||||
requests are cloned or retained.
|
||||
2. Add `RepairAttempts int` to `contracts.StructuredCompletionResponse`. It is
|
||||
the actual number of corrective generation calls, not the configured budget
|
||||
and not the number of total candidates.
|
||||
3. Validate a non-nil request value as zero through three at the adapter
|
||||
boundary so programmatic callers cannot bypass later file/config validation.
|
||||
4. When the request value is nil, leave `promptkit.RunRequest.Validation` nil
|
||||
so the prompt's complete contract remains authoritative.
|
||||
5. When the value is non-nil:
|
||||
- call `Engine.InspectPrompt(ctx, promptID, promptVersion)`;
|
||||
- copy `PromptInspection.OutputContract` by value;
|
||||
- replace only `RepairAttempts`;
|
||||
- pass the complete copied contract as `RunRequest.Validation`; and
|
||||
- prepare and execute exactly as before.
|
||||
Do not infer or hard-code schema paths, validation modes, or formats. Do not
|
||||
cache inspection in this stage.
|
||||
6. Map `result.Validation.RepairAttempts` to the response and leave
|
||||
`result.Usage` cumulative values unchanged. The existing debug validation
|
||||
object and prepared output contract should show actual and configured values
|
||||
respectively.
|
||||
7. Preserve result semantics:
|
||||
- valid initial and repaired candidates decode normally;
|
||||
- repair exhaustion returns the final raw candidate/debug material with an
|
||||
error matching `ErrInvalidStructuredOutput`;
|
||||
- explicit empty or whitespace-only content follows PromptKit validation;
|
||||
- missing/null/non-string content remains a generation/provider failure;
|
||||
- corrective-call generation errors use Stage 4's application-owned mapping;
|
||||
and
|
||||
- context cancellation wins at every error boundary.
|
||||
8. Keep `CompleteStructured` and its helpers provider neutral outside this
|
||||
adapter package. Do not expose PromptKit validation or inspection types.
|
||||
9. Update only the adapter-owned repair behavior in `docs/internal/llm.md` and
|
||||
`docs/integrations/pkg-promptkit.md`. State that public pipeline configuration
|
||||
and the production default are added by later stages of this plan.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Add adapter-level behavioral tests using a deterministic fake PromptKit LLM:
|
||||
|
||||
- nil override uses the prompt declaration;
|
||||
- explicit zero overrides a positive prompt declaration without dropping its
|
||||
JSON Schema contract;
|
||||
- explicit one turns an invalid first candidate followed by a valid candidate
|
||||
into one successful response with the final raw bytes, actual repair count
|
||||
one, and cumulative usage;
|
||||
- repair exhaustion returns the final candidate and validation diagnostics as
|
||||
`ErrInvalidStructuredOutput`;
|
||||
- explicit empty content is eligible for repair;
|
||||
- a corrective generation failure maps through Stage 4; and
|
||||
- invalid direct values below zero or above three fail before provider work.
|
||||
|
||||
Do not assert PromptKit's exact assistant/user correction prose or copy its
|
||||
full internal repair matrix.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The adapter changes only repair count when applying a request override.
|
||||
- Nil and explicit zero remain distinct.
|
||||
- Repaired success returns final raw output, cumulative usage, and actual count
|
||||
without a warning.
|
||||
- Exhaustion, empty content, corrective generation failure, and cancellation
|
||||
match the target semantics.
|
||||
- No PromptKit type crosses the LLM package boundary.
|
||||
|
||||
## Stage 6: Propagate Repair Policy Through Framework Requests ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Carry an optional effective repair budget from each resolved stage or validator
|
||||
binding to its module request without changing public file configuration yet.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` alongside `LLMProfile` to every
|
||||
stage request that can belong to an LLM-backed binding:
|
||||
- `ParseRequest`;
|
||||
- `ChunkRequest`;
|
||||
- `TypedExtractionRequest`;
|
||||
- `TypedMergeRequest`;
|
||||
- `TypedNormalizeRequest`;
|
||||
- `OutputRequest`;
|
||||
- `TypedValidationRequest`;
|
||||
- `ChunkValidationRequest`; and
|
||||
- `SerializedValidationRequest`.
|
||||
2. Add the same optional field to the erased/internal request carriers used by
|
||||
registry builders, preparation, runner stage attempts, validator targets,
|
||||
retry closures, and debug wrappers. Copy pointer values; never share a
|
||||
mutable pointer owned by configuration.
|
||||
3. At every runner stage invocation, obtain the value from the exact resolved
|
||||
producer binding. At every validator invocation, obtain it from that exact
|
||||
resolved validator binding. Do not use the producer's value for a validator
|
||||
or vice versa.
|
||||
4. Ensure all retry attempts for the same binding receive the same effective
|
||||
structural-repair value. Do not decrement it in Notarius; PromptKit owns the
|
||||
inner budget independently on each `CompleteStructured` call.
|
||||
5. Extend `semanticreconcile.Request` with the optional field and carry it into
|
||||
each generic reconciliation completion. A batched reconciliation may make
|
||||
several completion calls; each call receives the same effective budget.
|
||||
6. Update registry erasure/adaptation code for typed merge, normalize, and
|
||||
validation requests so no field is lost. Preserve input/output support even
|
||||
though current production input and output modules are deterministic.
|
||||
7. Add focused framework tests for one chunk producer, one extraction
|
||||
producer, one normalizer, and one LLM-backed validator. Verify exact pointer
|
||||
value propagation and separation between producer and validator settings.
|
||||
Do not add repetitive tests for every generic adapter.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/pipeline ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test -race ./internal/framework/pipeline ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every stage and validator request can carry a detached optional repair value.
|
||||
- The runner sources the value from the exact resolved binding.
|
||||
- Producer and validator values cannot overwrite one another.
|
||||
- Stage retries reuse but do not mutate or consume the inner repair budget.
|
||||
- Semantic reconciliation forwards the budget to every one of its completion
|
||||
calls.
|
||||
- Existing behavior remains unchanged while all values are nil.
|
||||
|
||||
## Stage 7: Forward Repair Policy From Every LLM-Backed Module ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Complete the internal end-to-end path by having every production LLM-backed
|
||||
module forward its stage request value to `CompleteStructured`.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inventory every production `CompleteStructured` call with code search before
|
||||
editing. The expected current owners include:
|
||||
- `dnd/scenes` chunking;
|
||||
- the combat-turn, enemy-event, item-occurrence, item-registry,
|
||||
location-occurrence, location-registry, NPC-occurrence, NPC-registry,
|
||||
scene-description, and spell extractors; and
|
||||
- generic semantic reconciliation used by the item, location, and NPC
|
||||
registry normalizers.
|
||||
Reconcile this list with the actual repository; do not omit a newly added
|
||||
production caller merely because it is not named here.
|
||||
2. In each direct caller, set
|
||||
`StructuredCompletionRequest.StructuredOutputRepairAttempts` from the
|
||||
corresponding stage request. Clone the pointer or use a small shared helper
|
||||
if that reduces repeated ownership mistakes without moving domain logic.
|
||||
3. Ensure D&D registry normalizers pass their typed normalize request value into
|
||||
`semanticreconcile.Request`, and that the generic engine forwards it as
|
||||
established in Stage 6.
|
||||
4. Update existing module prompt-mapping tests that already inspect a captured
|
||||
structured-completion request to assert the new field. Do not create a new
|
||||
one-test-per-module suite solely to memorialize field plumbing; rely on the
|
||||
existing request-contract tests plus a final complete call-site audit.
|
||||
5. Search again after editing for production `CompleteStructured` calls and
|
||||
verify each either forwards the field or documents why it cannot receive a
|
||||
pipeline binding. Test-only fakes need only preserve the field when their
|
||||
contract test depends on it.
|
||||
6. Do not set a module-specific fallback value. Nil must reach the adapter so
|
||||
the prompt declaration remains authoritative.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run focused D&D and semantic-reconciliation packages, then the full suite:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/modules/dnd/... ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test -race ./internal/modules/dnd/... ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every production LLM-backed completion receives the exact stage or validator
|
||||
repair value.
|
||||
- No module invents a default or imports PromptKit.
|
||||
- Registry normalizers preserve the value through semantic reconciliation.
|
||||
- Existing request-contract tests remain concise and pass.
|
||||
- A final call-site audit finds no silent production omission.
|
||||
|
||||
## Stage 8: Add The Public Repair Configuration Contract ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Add presence-aware pipeline and binding configuration for
|
||||
`structured_output_repair_attempts` without yet changing runtime resolution.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` to
|
||||
`pipeline.PipelineProfile` and `pipeline.ModuleBinding`, using
|
||||
`json:"structured_output_repair_attempts,omitempty"`.
|
||||
2. Add presence-aware YAML support at pipeline and object-binding scope:
|
||||
- exact key `structured_output_repair_attempts`;
|
||||
- integer values zero through three;
|
||||
- explicit null, non-integer, and out-of-range values rejected with scoped
|
||||
diagnostics; and
|
||||
- scalar shorthand bindings continue to omit the binding override.
|
||||
Preserve file configuration version 4.
|
||||
3. Update every configuration clone, conversion, redaction, summary, and JSON
|
||||
round-trip carrier. Copy pointers by value into newly allocated storage so
|
||||
parsed, configured, and redacted values do not alias.
|
||||
4. Preserve omission versus explicit zero through YAML parsing, profile
|
||||
inheritance, module-binding object form, and JSON round trips. Keep scalar
|
||||
shorthand bindings equivalent to omission.
|
||||
5. Do not add a top-level `promptkit.repair_attempts` setting or CLI override.
|
||||
6. Add concise parser and ownership tests. Defer execution-class checks,
|
||||
effective precedence, resolved digests, and runtime forwarding to Stage 9,
|
||||
where module metadata is available.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
At the parser/config boundary, test omitted, explicit zero, positive bounds,
|
||||
negative, above-three, null, non-integer, scalar shorthand, cloning, redaction,
|
||||
and JSON round-trip behavior. Use relational boundary tests for the allowed
|
||||
range and avoid duplicating the same cases at every layer.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/core/config ./internal/cli
|
||||
GOWORK=off go test -race ./internal/core/config
|
||||
GOWORK=off go test ./...
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The exact public field parses at pipeline and object-binding scope with the
|
||||
fixed range.
|
||||
- Nil and explicit zero remain distinguishable through parsing, cloning,
|
||||
inheritance, redaction, summaries, and round trips.
|
||||
- Both maintained configurations remain valid without requiring the new field.
|
||||
- No runtime or prompt default has changed prematurely.
|
||||
|
||||
## Stage 9: Resolve And Apply Repair Configuration ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Resolve the public field against module execution classes, incorporate the
|
||||
effective value into pipeline identity, and connect it to the request plumbing
|
||||
completed in Stages 6 and 7.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. During resolution, compute the effective value for every selected binding:
|
||||
- explicit binding value wins;
|
||||
- otherwise an explicit pipeline value applies to an LLM-backed binding;
|
||||
- otherwise leave nil for prompt-owned policy.
|
||||
Apply the pipeline value to LLM-backed validators as well as producers.
|
||||
2. Reject an explicit binding value on a deterministic module or deterministic
|
||||
validator using the same execution-class knowledge used for `llm_profile`.
|
||||
Do not reject a pipeline-level value merely because a selected pipeline also
|
||||
contains deterministic bindings; simply do not apply it to those bindings.
|
||||
3. Clone every resolved pointer so the parsed pipeline, resolved profile,
|
||||
redacted summaries, and runner requests have distinct ownership.
|
||||
4. Include the effective field in resolved pipeline JSON and digest input. A
|
||||
change between nil, zero, and a positive value must change the resolved
|
||||
digest when it changes an LLM-backed selected binding. Unselected lanes must
|
||||
retain the repository's existing digest and selection semantics.
|
||||
5. Pass the resolved value into the Stage 6 request field for every selected
|
||||
input, chunk, extract, merge, normalize, output, and validator binding.
|
||||
6. Update `docs/config.md` as the canonical field, range, and precedence
|
||||
contract; `docs/internal/pipeline.md` as the resolution owner; and
|
||||
`docs/operations.md` for the distinction from binding `retries`. The
|
||||
prompt-owned production default remains unchanged until Stage 10.
|
||||
7. Add focused resolution and runner tests. Cover representative execution
|
||||
classes rather than repeating the same assertion for every module type.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Test:
|
||||
|
||||
- binding over pipeline over nil precedence;
|
||||
- inheritance into each selected LLM-backed stage and validator;
|
||||
- no inheritance into deterministic bindings;
|
||||
- explicit deterministic-binding rejection;
|
||||
- detached pointers;
|
||||
- runner forwarding for representative producer and validator bindings; and
|
||||
- digest changes for execution-relevant nil, zero, and positive changes.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/pipeline ./internal/cli
|
||||
GOWORK=off go test -race ./internal/framework/pipeline
|
||||
GOWORK=off go test ./...
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The exact public field has the fixed binding-over-pipeline-over-prompt
|
||||
precedence for every selected LLM-backed producer and validator.
|
||||
- Deterministic binding misuse fails during resolution before execution, while
|
||||
a pipeline value coexists with deterministic bindings.
|
||||
- Nil and explicit zero remain distinguishable through resolution, runtime,
|
||||
summaries, and digests.
|
||||
- A policy change invalidates checkpoint identity when it changes an effective
|
||||
selected binding.
|
||||
- Current configuration, pipeline, and operations documentation matches the
|
||||
implemented behavior.
|
||||
|
||||
## Stage 10: Enable The Default, Finish Documentation, And Verify The Feature ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Set the accepted production default of one repair, reconcile all canonical
|
||||
documentation, and run the full repository verification pass.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Change `repair_attempts: 0` to `repair_attempts: 1` in every maintained
|
||||
production prompt manifest that produces structured output, including the
|
||||
generic semantic-reconciliation prompt and every D&D chunk, extraction, and
|
||||
registry-normalization prompt. Do not mechanically change unrelated test
|
||||
fixtures whose purpose is to exercise zero.
|
||||
2. Inspect every production prompt output contract after the edit. Confirm that
|
||||
each positive budget uses `basic`, `json`, or `json_schema`, remains no
|
||||
greater than three, and retains its existing format and schema path.
|
||||
3. Add or refine the smallest durable assembled-assets test that proves the
|
||||
production engine can prepare the maintained prompts with the activated
|
||||
contracts. Do not add a brittle test that asserts an exact prompt count,
|
||||
file count, message prose, correction text, or asset length. The public
|
||||
default may be tested at one canonical assembled boundary because its
|
||||
literal value is an operational contract.
|
||||
4. Confirm a successful repair does not create a warning and that exhausted
|
||||
repair remains `ErrInvalidStructuredOutput`. Verify the debug prompt records
|
||||
the configured contract, the debug response records actual repair count,
|
||||
and cumulative usage is not double-counted.
|
||||
5. Confirm scheduling behavior with one focused test or existing coverage: the
|
||||
Notarius scheduled client admits one logical `CompleteStructured` operation
|
||||
while PromptKit may make serial corrective provider calls inside it. Do not
|
||||
attempt to reacquire a Notarius permit from inside PromptKit or add a second
|
||||
scheduler.
|
||||
6. Reconcile current-state documentation:
|
||||
- `docs/integrations/pkg-promptkit.md` owns the pinned upstream boundary;
|
||||
- `docs/config.md` owns field names, range, default, and precedence;
|
||||
- `docs/operations.md` owns latency/cost, optional credentials, concurrency,
|
||||
timeout, and the upper-bound formula;
|
||||
- `docs/internal/llm.md` owns inspection-based contract replacement,
|
||||
cumulative usage, actual repair count, generation errors, and debug data;
|
||||
- `docs/internal/pipeline.md` owns effective policy propagation and the
|
||||
separation from stage retries; and
|
||||
- `docs/policy/architecture.md` adds only the durable rule that PromptKit
|
||||
owns deterministic structural repair inside one completion while Notarius
|
||||
owns stage attempts and semantic validation.
|
||||
7. Remove current-behavior claims that PromptKit is v0.5.0, that every
|
||||
production repair budget is zero, or that PromptKit is always single-pass.
|
||||
Do not alter historical release notes or archived roadmaps.
|
||||
8. Keep the maintained minimal and complete examples secret-free and valid.
|
||||
They may omit the new field to demonstrate the default; do not add a
|
||||
redundant complete profile or Rakestrawhome example merely to exercise an
|
||||
upstream catalog entry.
|
||||
9. Review `docs/roadmap/future.md` only for consistency. Leave the future
|
||||
feedback-aware stage retry, combat-scene validator, and warning-reform work
|
||||
unimplemented and clearly separate.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run focused tests first, then all repository checks:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/framework/pipeline ./internal/framework/semanticreconcile ./internal/modules/dnd/...
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./cmd/notarius
|
||||
GOWORK=off go mod tidy -diff
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Also perform focused repository searches that exclude `docs/roadmap/archive/`
|
||||
and historical release notes:
|
||||
|
||||
- no active v0.5.0 PromptKit pins or links remain;
|
||||
- no maintained production prompt still declares `repair_attempts: 0`;
|
||||
- every production `CompleteStructured` caller forwards the repair field; and
|
||||
- no provider code, type, or message is added to ordinary errors, warnings,
|
||||
manifests, cache, or checkpoint schemas.
|
||||
|
||||
If the repository's source-release checker is available and the ordinary
|
||||
checks above pass, run `./scripts/check-release-source.sh v0.0.0` as the final
|
||||
integrated validation. It must not create a tag, release note, or repository
|
||||
artifact.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every maintained structured prompt defaults to one corrective call and can
|
||||
be overridden to zero through three at pipeline or binding scope.
|
||||
- A real assembled Notarius completion follows the PromptKit v0.8.0 repair
|
||||
contract without changing prompt schema identity or cacheable prefix.
|
||||
- Actual repair count, cumulative usage, error classification, debug-only
|
||||
provider diagnostics, scheduling, and checkpoint identity match the feature
|
||||
roadmap.
|
||||
- Profile inheritance, Rakestrawhome availability, optional credentials, and
|
||||
v0.6.0 hardening remain covered and documented.
|
||||
- All canonical documentation describes implemented v0.8.0 behavior in its
|
||||
assigned home and leaves future semantic validation work in the roadmap.
|
||||
- Maintained examples validate, all ordinary/race/vet/build/module checks pass,
|
||||
and the worktree contains no generated or sensitive artifacts.
|
||||
520
docs/roadmap/promptkit-v0.8.md
Normal file
520
docs/roadmap/promptkit-v0.8.md
Normal file
@@ -0,0 +1,520 @@
|
||||
# PromptKit v0.8.0 Upgrade
|
||||
|
||||
## Status
|
||||
|
||||
Proposed.
|
||||
|
||||
## Purpose
|
||||
|
||||
Upgrade Notarius from PromptKit v0.5.0 to v0.8.0 and deliberately adopt the
|
||||
useful correctness, profile-composition, provider-diagnostic, backend, and
|
||||
structured-output-repair capabilities introduced in PromptKit v0.6.0, v0.7.0,
|
||||
and v0.8.0.
|
||||
|
||||
The upgrade should improve structured-output reliability without confusing
|
||||
PromptKit's bounded deterministic repair with Notarius's existing stage retry
|
||||
budget or the future feedback-aware semantic-validation loop. PromptKit types
|
||||
and provider behavior must remain behind Notarius's transport-neutral LLM
|
||||
boundary.
|
||||
|
||||
## Current State
|
||||
|
||||
Notarius currently pins PromptKit v0.5.0. Its production adapter prepares one
|
||||
frozen execution, records credential-redacted details, and runs that same
|
||||
prepared value. It maps PromptKit capacity failures to an application-owned
|
||||
error, maps failed structured validation to `ErrInvalidStructuredOutput`, and
|
||||
returns PromptKit's raw validated bytes and usage metadata.
|
||||
|
||||
Every maintained production prompt uses JSON Schema validation and currently
|
||||
declares `repair_attempts: 0`. Notarius stage bindings separately expose
|
||||
`retries`, which reruns a complete stage operation after an error or rejected
|
||||
candidate. The two mechanisms have different ownership and must remain
|
||||
independent.
|
||||
|
||||
Notarius also maintains:
|
||||
|
||||
- embedded prompt, schema, and fallback-profile filesystems;
|
||||
- operator profile-file and profile-directory sources;
|
||||
- one optional conventional `local` backend registration;
|
||||
- explicit profile preflight through PromptKit inspection;
|
||||
- one application-wide scheduled LLM client around the PromptKit adapter;
|
||||
- PromptKit profile-source fingerprints for checkpoint safety; and
|
||||
- redacted debug and manifest provenance at application-owned boundaries.
|
||||
|
||||
The upgrade must preserve those established responsibilities while revising
|
||||
the pinned integration contract and any behavior affected by the three
|
||||
intervening releases.
|
||||
|
||||
This roadmap is based on PromptKit's pinned release guides for
|
||||
[v0.6.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.6.0.md),
|
||||
[v0.7.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.7.0.md),
|
||||
and
|
||||
[v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.8.0.md),
|
||||
plus the public API and format documentation at the v0.8.0 tag.
|
||||
|
||||
## Target End State
|
||||
|
||||
- `go.mod` and `go.sum` pin PromptKit v0.8.0 without a local replacement or
|
||||
vendored copy.
|
||||
- Every maintained PromptKit prompt and profile prepares successfully under
|
||||
v0.8.0's stricter validation and source-loading rules.
|
||||
- Eligible Notarius structured completions use one PromptKit corrective call by
|
||||
default after a structurally invalid response. Operators can explicitly set
|
||||
a value from zero through three for a configured pipeline, with a more local
|
||||
LLM-backed binding override where needed.
|
||||
- PromptKit repair remains an inner operation within one Notarius stage
|
||||
attempt. It never consumes or replenishes the binding's `retries` budget.
|
||||
- A successful repaired result exposes cumulative usage and the actual repair
|
||||
count to Notarius's application-owned response and debug models. A repaired
|
||||
success is not itself a warning.
|
||||
- Exhausted PromptKit validation remains an invalid structured-output result,
|
||||
preserving the final candidate and diagnostics for debug and for any
|
||||
applicable outer Notarius stage policy. Invalid structured output is never
|
||||
accepted merely because the repair budget was exhausted.
|
||||
- Profile inheritance, the built-in Rakestrawhome backend/profile, optional
|
||||
credential behavior, and structured generation errors work through the
|
||||
existing Notarius PromptKit boundary and are accurately documented.
|
||||
- Provider-specific PromptKit types do not escape `internal/framework/llm`.
|
||||
- Checkpoint identity, effective configuration, redacted summaries, and debug
|
||||
provenance reflect every execution-affecting repair or profile change.
|
||||
- Current documentation pins and describes v0.8.0; future Notarius semantic
|
||||
validation retries remain roadmap behavior rather than being conflated with
|
||||
this dependency upgrade.
|
||||
|
||||
## Release-by-Release Adoption
|
||||
|
||||
### PromptKit v0.6.0: Correctness, Safety, And Efficiency
|
||||
|
||||
PromptKit v0.6.0 adds no public declarations, but intentionally rejects several
|
||||
formerly permissive or ambiguous inputs. The upgrade must audit Notarius's
|
||||
embedded and operator-facing integration against these rules:
|
||||
|
||||
- YAML `id` and `version` metadata, rather than filenames, define prompt and
|
||||
profile identity.
|
||||
- Prompt `content_file` paths are exact, relative, contained paths; built-in
|
||||
file artifacts must resolve to regular files.
|
||||
- execution controls, output contracts, and repair budgets must be finite and
|
||||
within their documented ranges;
|
||||
- provider endpoints must be absolute HTTP or HTTPS URLs with a host and no
|
||||
user information, query, or fragment;
|
||||
- JSON documents and successful provider responses contain exactly one value;
|
||||
- successful provider responses are bounded to 16 MiB; and
|
||||
- JSON-compatible values are bounded for depth and expansion.
|
||||
|
||||
Notarius should rely on PromptKit for these rules rather than duplicate its
|
||||
parsers or internal limits. Existing Notarius validation may retain a narrower
|
||||
application rule where it has independent value, but overlapping validation
|
||||
must agree with PromptKit and must not accept a value PromptKit will reject
|
||||
later.
|
||||
|
||||
The upgrade automatically receives operation-local schema-plan reuse,
|
||||
artifact-text memoization, improved cancellation checks, and transport error
|
||||
identity preservation. Notarius should verify these changes through its real
|
||||
adapter boundary and avoid adding a second cache or response-body layer that
|
||||
would duplicate PromptKit's ownership.
|
||||
|
||||
### PromptKit v0.7.0: Profiles, Backend Access, And Generation Errors
|
||||
|
||||
#### Profile Inheritance
|
||||
|
||||
Operator profiles may use `base_profile` to alias or selectively refine a
|
||||
built-in, fallback, or higher-precedence operator profile. Notarius must pass
|
||||
profile sources through unchanged and let PromptKit own parent lookup, merge
|
||||
rules, source precedence, cycle detection, and fully resolved prepared targets.
|
||||
|
||||
Preflight inspection must resolve inherited profiles through the same source
|
||||
and backend composition used at execution. The selected leaf profile ID remains
|
||||
the public profile identity, while effective backend, endpoint, model, and
|
||||
reasoning provenance reflect the resolved chain. Notarius must not implement a
|
||||
second inheritance parser.
|
||||
|
||||
The existing complete `dnd-extraction` fallback remains a standalone profile:
|
||||
PromptKit v0.8.0 does not provide a built-in `openai/gpt-5.6-luna` profile that
|
||||
would be an appropriate parent. Documentation should nevertheless explain how
|
||||
operators can use inheritance for environment-specific workload profiles and
|
||||
should link to PromptKit's pinned format contract rather than duplicate its
|
||||
field-by-field merge algorithm.
|
||||
|
||||
Checkpoint safety must cover inherited behavior. Operator file/directory
|
||||
digests already cover changes to definitions in those sources, fallback asset
|
||||
digests cover application parents, and the PromptKit built-in catalog marker
|
||||
must change from its v0.5.0 identity to v0.8.0 so a changed built-in parent
|
||||
cannot reuse an incompatible checkpoint.
|
||||
|
||||
#### Rakestrawhome Backend And Profile
|
||||
|
||||
PromptKit's reserved `rakestrawhome` backend and
|
||||
`rakestrawhome-gemma-4-31b` profile become available without Notarius-specific
|
||||
registration. Notarius must not register or shadow the reserved backend ID.
|
||||
Profile preflight, backend-capacity reporting, scheduling, generation, and
|
||||
provenance should work for it through the same generic paths used by OpenRouter
|
||||
and `local`.
|
||||
|
||||
The D&D default remains `dnd-extraction`; this upgrade does not silently move a
|
||||
production workload to Rakestrawhome. Operator documentation should identify
|
||||
the built-in profile as an available selection and link to PromptKit for its
|
||||
endpoint, credential environment, model, and capacity defaults.
|
||||
|
||||
#### Optional Credentials
|
||||
|
||||
An absent or blank optional `APIKeyEnv` now causes PromptKit to omit the
|
||||
`Authorization` header and send the request. Notarius must not restore the old
|
||||
failure behavior by pre-reading provider credential environment variables or
|
||||
by adding provider-specific authentication logic.
|
||||
|
||||
Profile inspection may report an explicit `APIKeyRequired` policy without
|
||||
reading the credential, and execution remains the boundary at which that
|
||||
requirement is enforced. For optional profiles, an authentication-requiring
|
||||
provider may instead return a structured 401 or 403 generation failure. The
|
||||
configuration and operations documentation must explain this distinction.
|
||||
Notarius does not currently expose PromptKit's in-memory profile-registration
|
||||
API to operators, and PromptKit's filesystem profile format does not expose
|
||||
`APIKeyRequired`; therefore Notarius must not promise that an operator profile
|
||||
can force local credential preflight. Operators should provision the named
|
||||
environment variable, while Notarius should preserve the provider's structured
|
||||
authentication failure when it is absent.
|
||||
|
||||
Notarius must continue to document mechanisms and environment-variable names,
|
||||
never secret values.
|
||||
|
||||
#### Structured Generation Errors
|
||||
|
||||
The adapter should recognize `*promptkit.GenerationError` with `errors.As` and
|
||||
translate useful information into an immutable, provider-neutral Notarius
|
||||
error classification. At minimum, retain the HTTP status code so callers and
|
||||
future retry policy can distinguish transport success with provider rejection
|
||||
from other generation failures.
|
||||
|
||||
PromptKit's provider code, type, and message accessors are bounded but remain
|
||||
untrusted and potentially sensitive. They must never appear automatically in
|
||||
ordinary CLI output, warnings, manifests, checkpoint identity, or cache data.
|
||||
If retained for an explicitly requested debug trace, they must pass through
|
||||
Notarius's known-secret and bearer redaction and remain clearly identified as
|
||||
untrusted provider diagnostics. Default error formatting should continue to
|
||||
use a bounded, redacted application-owned message.
|
||||
|
||||
Capacity and cancellation retain their current more specific classifications
|
||||
and precedence. This upgrade does not add automatic provider-error retry
|
||||
classification; it only preserves safe structured data needed for diagnosis
|
||||
and later policy.
|
||||
|
||||
### PromptKit v0.8.0: Bounded Structured-Output Repair
|
||||
|
||||
#### Default Policy
|
||||
|
||||
Every maintained production prompt whose output is consumed as structured data
|
||||
should declare one repair attempt. All current production prompts use eligible
|
||||
JSON Schema validation, so no current prompt needs a zero default merely
|
||||
because of its output mode.
|
||||
|
||||
One repair means at most one corrective generation after the initial
|
||||
candidate. PromptKit reconstructs the immutable original conversation and
|
||||
appends only the latest invalid assistant candidate and latest deterministic
|
||||
validation diagnostics. It preserves the selected target, direct session ID,
|
||||
provider-native structured-output contract, and backend capacity policy. This
|
||||
shape preserves the original cacheable prompt prefix and avoids accumulating
|
||||
unbounded failed history.
|
||||
|
||||
The default is deliberately small. A single repair captures the common case in
|
||||
which a capable model can correct malformed JSON or a schema violation after
|
||||
receiving an exact diagnostic, while bounding the extra latency and cost of a
|
||||
single structured completion.
|
||||
|
||||
#### Configuration Contract
|
||||
|
||||
The public configuration is an optional, presence-aware
|
||||
`structured_output_repair_attempts` integer at pipeline scope and at each
|
||||
LLM-backed module or validator binding. Its effective precedence is:
|
||||
|
||||
1. the binding value, when present;
|
||||
2. the pipeline value, when present; and
|
||||
3. the selected prompt's declared `repair_attempts` value.
|
||||
|
||||
The value must be from zero through three. Explicit zero disables PromptKit
|
||||
repair at that scope. A deterministic binding must reject the field because it
|
||||
cannot perform structured LLM repair. Validator bindings may use it only when
|
||||
the selected validator is LLM-backed. Shorthand module bindings continue to
|
||||
inherit the pipeline or prompt default.
|
||||
|
||||
The long, provider-neutral name is intentional: it distinguishes PromptKit's
|
||||
inner structural repair from the existing binding `retries` field, which owns
|
||||
complete stage attempts, without exposing a dependency name in generic
|
||||
pipeline contracts.
|
||||
|
||||
The effective value must survive file parsing, cloning, redacted summaries,
|
||||
pipeline resolution, and pipeline digest construction without pointer aliasing
|
||||
or loss of presence. It must affect checkpoint identity because it can change
|
||||
the selected result, latency, token usage, and provider cost.
|
||||
|
||||
#### Adapter Contract
|
||||
|
||||
The transport-neutral structured-completion request should carry an optional
|
||||
application-owned structural-repair budget. No `promptkit.OutputContract` or
|
||||
other PromptKit type may cross the adapter boundary.
|
||||
|
||||
PromptKit v0.8.0 request validation replaces the complete prompt output
|
||||
contract rather than merging one field. When Notarius has a configured
|
||||
override, the adapter must therefore inspect the selected prompt, copy its
|
||||
normalized declared format, validation mode, and schema path, change only the
|
||||
repair count, and supply that complete contract on the prepared request. A nil
|
||||
override continues to use the prompt declaration directly. Inspection and
|
||||
preparation must use the same immutable engine sources; a small adapter-local
|
||||
cache keyed by normalized prompt ID and version is acceptable but not required
|
||||
without measured need.
|
||||
|
||||
This approach prevents configuration from accidentally dropping JSON Schema
|
||||
validation, avoids duplicating schema paths in pipeline YAML, and keeps prompt
|
||||
assets authoritative for every output-contract field other than the explicit
|
||||
operator override.
|
||||
|
||||
The transport-neutral structured-completion response should report the actual
|
||||
number of PromptKit repair calls. PromptKit's returned token usage is already
|
||||
cumulative and must be passed through without re-summing it. Debug records
|
||||
should distinguish the configured budget from the actual count. Ordinary run
|
||||
manifests need not gain raw prompt or response data merely to report repairs;
|
||||
any durable aggregate should be added only if it has a clear consumer contract.
|
||||
|
||||
#### Result And Failure Semantics
|
||||
|
||||
- A valid initial candidate returns normally with zero actual repairs.
|
||||
- A valid corrected candidate returns normally with cumulative usage and its
|
||||
positive actual repair count. It does not emit a warning solely because a
|
||||
repair occurred.
|
||||
- Exhausting the repair budget returns PromptKit's final candidate and failed
|
||||
validation result. The adapter maps this to
|
||||
`ErrInvalidStructuredOutput`, preserves the response and debug material, and
|
||||
does not decode or accept the candidate.
|
||||
- An explicitly empty or whitespace-only candidate participates in the
|
||||
declared structural validation and repair flow. Missing, `null`, or
|
||||
non-string provider content remains a malformed provider response.
|
||||
- A generation failure during a corrective call is an operational generation
|
||||
failure and uses the same safe structured-error adaptation as an initial
|
||||
generation failure.
|
||||
- Context cancellation remains authoritative throughout the initial and
|
||||
corrective calls.
|
||||
|
||||
PromptKit repair happens inside one scheduled `CompleteStructured` operation.
|
||||
The Notarius scheduler holds one permit for that logical operation while
|
||||
PromptKit performs its initial and serial corrective calls; PromptKit
|
||||
reacquires its own selected-backend capacity for each corrective generation.
|
||||
Because corrective calls are serial, this cannot expand actual concurrent
|
||||
provider work beyond the number of admitted Notarius operations, but
|
||||
documentation must stop describing the Notarius permit as a separate admission
|
||||
event for every internal repair call.
|
||||
|
||||
One `CompleteStructured` invocation with effective PromptKit repair budget `R`
|
||||
may make at most `R + 1` provider calls. If one stage attempt makes `C`
|
||||
structured-completion invocations, a binding with `retries: N` has an upper
|
||||
bound of `(N + 1) * C * (R + 1)` provider calls; `C` may itself be a bounded,
|
||||
data-dependent module property, as it is for batched semantic reconciliation.
|
||||
LLM-backed validators have their own corresponding invocation counts, budgets,
|
||||
and costs. These formulas are upper bounds, not promises that every failure is
|
||||
retryable or that every attempt reaches the provider.
|
||||
|
||||
## Profile And Prompt Source Compatibility
|
||||
|
||||
The upgrade must preserve Notarius's source precedence: an operator source,
|
||||
then registered application fallback profiles, then PromptKit built-ins. A
|
||||
selected malformed definition remains authoritative and fails rather than
|
||||
falling through. Parent resolution introduced by profile inheritance observes
|
||||
that same precedence.
|
||||
|
||||
All embedded prompt manifests, shared content fragments, response schemas, and
|
||||
fallback profiles must be prepared or inspected offline under v0.8.0. The
|
||||
review should specifically catch:
|
||||
|
||||
- IDs inferred accidentally from filenames;
|
||||
- stale or escaping `content_file` paths;
|
||||
- missing or non-regular embedded artifacts;
|
||||
- repair values outside zero through three or paired with ineligible
|
||||
validation;
|
||||
- schemas or examples that are not exact single JSON documents;
|
||||
- unsupported endpoint forms; and
|
||||
- JSON-compatible variables or profile extras that exceed upstream bounds.
|
||||
|
||||
No prompt prose, schema shape, durable D&D artifact contract, or default D&D
|
||||
model should change merely to exercise the dependency. Prompt manifests should
|
||||
change only as needed to enable the adopted repair default and satisfy v0.8.0
|
||||
contracts.
|
||||
|
||||
## Provenance, Debugging, And Security
|
||||
|
||||
- Update the opaque PromptKit built-in profile-catalog identity from v0.5.0 to
|
||||
v0.8.0. Do not hash or publish PromptKit's internal catalog bytes.
|
||||
- Ensure a prompt's repair default remains covered by its existing prompt asset
|
||||
fingerprint and a configured effective override remains covered by the
|
||||
resolved pipeline digest.
|
||||
- Preserve selected leaf profile identity while recording the inherited
|
||||
effective target already exposed by PromptKit inspection and prepared
|
||||
details.
|
||||
- Add actual structural-repair count and, when useful, the configured budget to
|
||||
application-owned debug material. Token totals remain PromptKit's cumulative
|
||||
values.
|
||||
- Do not generate a warning for a successful repair. Repair exhaustion is an
|
||||
invalid-output failure, while provider rejection is a generation failure.
|
||||
- Never expose raw provider diagnostic fields without explicit debug capture
|
||||
and application redaction. Do not place them in normal errors or durable
|
||||
summaries.
|
||||
- Preserve context and transport error identity sufficiently for
|
||||
`errors.Is`-based cancellation and deadline handling after adapting the
|
||||
external error.
|
||||
|
||||
## Documentation And Examples
|
||||
|
||||
Implementation should update current-state documentation only when the new
|
||||
behavior lands:
|
||||
|
||||
- `docs/integrations/pkg-promptkit.md` must pin v0.8.0 and define the revised
|
||||
prepared-execution, repair, profile-inheritance, backend, credential, and
|
||||
error-adaptation boundary.
|
||||
- `docs/config.md` must own the repair configuration fields, precedence,
|
||||
allowed range, explicit-zero behavior, profile inheritance availability, and
|
||||
optional credential semantics.
|
||||
- `docs/operations.md` must explain structural repair cost, timeout and
|
||||
concurrency effects, credential failures, and its distinction from stage
|
||||
retries.
|
||||
- `docs/internal/llm.md` must describe adapter contract replacement, actual
|
||||
repair metadata, error adaptation, source compatibility, and scheduling.
|
||||
- `docs/internal/pipeline.md` must describe how effective repair configuration
|
||||
is resolved and how inner repair differs from outer stage attempts.
|
||||
- `docs/policy/architecture.md` should receive only the durable ownership rule:
|
||||
PromptKit owns bounded deterministic structural repair within one completion,
|
||||
while Notarius owns stage attempts and semantic validation policy. Detailed
|
||||
fields and retry formulas belong in their canonical configuration and
|
||||
operations documents.
|
||||
|
||||
Update maintained configuration examples only if the public Notarius
|
||||
configuration contract changes. A short inheritance illustration may remain in
|
||||
the configuration reference; do not create a complete example solely to copy
|
||||
PromptKit's upstream profile catalog. All upstream links must point to the
|
||||
v0.8.0 tag. Historical release or archived roadmap references should remain
|
||||
historical.
|
||||
|
||||
No ADR is required solely to pin a newer dependency. The durable separation
|
||||
between PromptKit structural repair and Notarius semantic stage retries should
|
||||
be stated in architecture documentation now; the more extensive future
|
||||
validation state machine still warrants the separate ADR already identified in
|
||||
`future.md` when that work is promoted.
|
||||
|
||||
## Validation And Acceptance Criteria
|
||||
|
||||
The implementation is complete when:
|
||||
|
||||
- the repository builds and tests against PromptKit v0.8.0 with no replacement
|
||||
directive, workspace dependency, or vendored source;
|
||||
- every maintained prompt and profile prepares or inspects successfully under
|
||||
the v0.8.0 source, path, endpoint, output-contract, and JSON-value rules;
|
||||
- an invalid first JSON Schema candidate followed by a valid correction returns
|
||||
the valid raw output, cumulative usage, and actual repair count through the
|
||||
Notarius adapter;
|
||||
- repair exhaustion returns the final raw candidate and debug material with an
|
||||
error matching `ErrInvalidStructuredOutput`;
|
||||
- a corrective generation failure retains safe generation classification and
|
||||
provider status without leaking untrusted provider detail;
|
||||
- explicit empty content follows structural validation rather than being
|
||||
misclassified by Notarius;
|
||||
- repair configuration is presence-aware, range checked, rejected on
|
||||
deterministic bindings, resolved with documented precedence, and included in
|
||||
effective pipeline identity;
|
||||
- inherited profiles resolve consistently during preflight and execution, and
|
||||
changes to any relevant operator, fallback, or built-in parent invalidate
|
||||
checkpoint reuse;
|
||||
- the Rakestrawhome built-in profile reaches generic preflight, scheduling, and
|
||||
provenance paths without application-specific registration;
|
||||
- optional missing credentials and explicitly required credentials behave as
|
||||
documented without contacting real providers in tests;
|
||||
- cancellation, timeout, backend capacity, prepared-execution snapshot,
|
||||
session ID, raw-output, debug-redaction, and existing profile provenance
|
||||
behavior remain intact;
|
||||
- maintained examples validate successfully; and
|
||||
- canonical documentation contains no active v0.5.0 pin or claim that PromptKit
|
||||
is always single-pass.
|
||||
|
||||
Tests should follow `docs/policy/testing.md`: exercise observable Notarius
|
||||
contracts with offline fake clients or `httptest` boundaries, and do not copy
|
||||
PromptKit's entire internal repair test suite or assert its exact correction
|
||||
message prose. The dependency's internal wording is not a Notarius contract.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Implementing Notarius's future feedback-aware semantic stage-retry loop.
|
||||
- Adding the D&D combat-scene semantic validator.
|
||||
- Redesigning warning policy or treating successful structural repair as a
|
||||
warning.
|
||||
- Adding provider transport retries or deciding which HTTP statuses should
|
||||
consume a stage retry.
|
||||
- Exposing PromptKit request, response, profile, validation, capacity, or error
|
||||
types outside the LLM adapter.
|
||||
- Changing durable artifact schemas, D&D prompt semantics, the D&D default
|
||||
model, or the fixed pipeline shape.
|
||||
- Reimplementing PromptKit profile inheritance, schema validation, response
|
||||
bounds, repair conversations, backend admission, or provider parsing inside
|
||||
Notarius.
|
||||
|
||||
## Decisions
|
||||
|
||||
### 1. Default Structured-Output Repair Budget
|
||||
|
||||
**Decision: default to one repair attempt.** Set every maintained
|
||||
eligible production prompt to `repair_attempts: 1`. One corrective call is a
|
||||
strong fit for Notarius because every current production LLM response has a
|
||||
strict JSON Schema contract, smaller cost-effective models are a deliberate
|
||||
deployment target, and a precise structural diagnostic often makes one retry
|
||||
materially more successful. The budget is paid only after a structurally
|
||||
invalid candidate and remains tightly bounded.
|
||||
|
||||
**Alternative considered: retain zero by default.** This preserves single-pass
|
||||
cost and latency and requires operators to opt in. It is preferable for an
|
||||
environment where every additional request is expensive or where upstream
|
||||
provider-native schema enforcement already produces negligible invalid output.
|
||||
It is less suitable as the Notarius default because one malformed response can
|
||||
otherwise discard substantial completed pipeline work.
|
||||
|
||||
**Alternative considered: default to two.** This may improve recovery for
|
||||
weak models, but it doubles the worst-case corrective cost relative to the
|
||||
selected default and compounds with outer stage retries. It should be an
|
||||
operator choice supported by configuration, not the initial default, unless
|
||||
observational evidence shows that the second correction has a worthwhile
|
||||
marginal success rate.
|
||||
|
||||
### 2. Repair Override Scope
|
||||
|
||||
**Decision: support both pipeline and LLM-backed binding overrides.** Use
|
||||
the presence-aware `structured_output_repair_attempts` field and precedence
|
||||
defined above. A pipeline value provides the convenient one-line control the
|
||||
operator requested, while a binding value permits an expensive normalizer or
|
||||
future LLM-backed validator to use a deliberately different budget. This
|
||||
mirrors Notarius's established pipeline/binding profile inheritance and scales
|
||||
without editing embedded prompts.
|
||||
|
||||
**Alternative considered: support only a pipeline override.** This is smaller to
|
||||
implement and document and still permits global enablement or disablement for
|
||||
one pipeline. Its drawback is that one exceptional prompt cannot opt out or
|
||||
request a larger budget without changing an embedded asset for every pipeline.
|
||||
|
||||
**Alternative considered: expose one global value under the top-level
|
||||
`promptkit` configuration.** This makes client construction simple, but applies
|
||||
the same budget to unrelated pipelines and leaks an execution policy into the
|
||||
dependency configuration block. It is less compositional than pipeline-owned
|
||||
policy and therefore not recommended.
|
||||
|
||||
### 3. Retention Of Provider-Supplied Generation Details
|
||||
|
||||
**Decision: retain status in the application-owned error contract and
|
||||
retain redacted provider code, type, and message only in explicitly requested
|
||||
debug traces.** Status is useful for diagnosis and future retry policy without
|
||||
usually containing sensitive data. The other fields can materially explain a
|
||||
400 response but may echo request or schema content, so they belong only in the
|
||||
already-sensitive debug surface after Notarius redaction.
|
||||
|
||||
**Alternative considered: retain only HTTP status and discard all provider fields.**
|
||||
This is the safest and smallest policy and still improves typed failure
|
||||
handling. It sacrifices potentially decisive provider diagnostics, leaving an
|
||||
operator with less information when a provider returns a terse status and the
|
||||
problem cannot be reproduced easily.
|
||||
|
||||
**Alternative considered: include bounded provider code and type in normal
|
||||
errors while keeping message debug-only.** Codes and types are often stable and
|
||||
less sensitive than messages, but PromptKit explicitly classifies every
|
||||
provider field as untrusted. Promoting them to ordinary output creates a
|
||||
disclosure and compatibility burden that is not currently justified.
|
||||
282
docs/roadmap/source-releases.md
Normal file
282
docs/roadmap/source-releases.md
Normal file
@@ -0,0 +1,282 @@
|
||||
# Source-Only Releases
|
||||
|
||||
## Status
|
||||
|
||||
Implemented. Creating the first release under this procedure remains a
|
||||
separate maintainer operation.
|
||||
|
||||
## Purpose
|
||||
|
||||
Define a repeatable, guarded release process for Notarius without taking on a
|
||||
binary-distribution system that its current operator audience does not need.
|
||||
The process should make an exact source revision, its compatibility impact,
|
||||
and its validation status easy to identify while keeping installation in the
|
||||
hands of technically capable operators and deployment automation.
|
||||
|
||||
The model is adapted from Weatherreporter's release procedure, but its target
|
||||
is deliberately narrower: an immutable source tag and checked-in release note
|
||||
are the release. Notarius does not publish executable archives or support
|
||||
Windows as part of this work.
|
||||
|
||||
## Release Model
|
||||
|
||||
Notarius releases come from commits on `main` and use stable semantic-version
|
||||
tags in the form `vMAJOR.MINOR.PATCH`. Prerelease tags are not part of the
|
||||
initial process.
|
||||
|
||||
Every release has one nonempty, version-matched note at
|
||||
`docs/releases/<tag>.md`. The note and every affected current-state document
|
||||
must be present in the tagged commit. The Git tag and checked-in note together
|
||||
are the durable release record; no separately editable release page is
|
||||
required.
|
||||
|
||||
Published tags are immutable. A maintainer must never move, reuse, or delete a
|
||||
published tag. If a published candidate is defective, the correction is made
|
||||
on `main` and released under a new patch version. An unpublished local tag may
|
||||
be deleted when candidate inspection finds a problem before any remote push.
|
||||
|
||||
Before `v1.0.0`, a minor release may intentionally change a documented CLI,
|
||||
configuration, durable artifact, integration, or operating contract when its
|
||||
release note explains the impact and required operator action. A patch release
|
||||
must not intentionally break those documented contracts within its minor
|
||||
line.
|
||||
|
||||
The existing `v0.1.0`, `v0.2.0`, and `v0.3.0` tags remain unchanged. They
|
||||
predate this procedure and do not need retrospective release notes. The first
|
||||
release made under this process establishes the release-note series.
|
||||
|
||||
## Source-Only Distribution
|
||||
|
||||
Notarius does not publish release binaries, archives, installers, container
|
||||
images, package-manager entries, checksum files, or signatures. A release tag
|
||||
is suitable for Go-native installation and for an operator-controlled build
|
||||
from an exact checkout.
|
||||
|
||||
The primary installation form is:
|
||||
|
||||
```sh
|
||||
GOWORK=off go install \
|
||||
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
Operator documentation should also describe cloning the repository, checking
|
||||
out the tag in detached-head state, and building `./cmd/notarius` with the Go
|
||||
version declared by `go.mod`. Private-module authentication and `GOPRIVATE`
|
||||
configuration belong to the operator environment and must be documented by
|
||||
mechanism rather than with real credentials.
|
||||
|
||||
Consumers such as Narratio should pin the desired Notarius tag in provisioning
|
||||
or deployment configuration. They must continue to decide runtime
|
||||
compatibility from Notarius's published receipt and artifact schema contracts,
|
||||
not merely from the executable's product version.
|
||||
|
||||
Packaged binaries may be reconsidered if distribution demand, installation
|
||||
friction, or a broader user audience justifies their build, signing, retention,
|
||||
and platform-support costs. They are not a prerequisite for a disciplined
|
||||
release process.
|
||||
|
||||
## Platform Policy
|
||||
|
||||
Linux is the supported deployment platform. Release validation must run the
|
||||
test suite and the release build on Linux and must confirm that the command
|
||||
builds with `CGO_ENABLED=0` for Linux `amd64` and `arm64`.
|
||||
|
||||
macOS is a best-effort development and testing platform. Release validation
|
||||
should confirm that the command cross-compiles with `CGO_ENABLED=0` for Darwin
|
||||
`amd64` and `arm64`, but the project does not promise packaged artifacts or a
|
||||
separate runtime test environment for those targets.
|
||||
|
||||
Windows is unsupported. The release process must not require Windows builds,
|
||||
Windows-specific compatibility work, or Windows documentation. Platform-
|
||||
specific implementation may intentionally use Unix facilities when they are
|
||||
important to Notarius's filesystem safety and operational model. Any later
|
||||
decision to support Windows requires its own feature scope and validation
|
||||
policy.
|
||||
|
||||
## Version Reporting
|
||||
|
||||
Add a root `notarius --version` interface for deployment diagnostics. It
|
||||
prints exactly one line:
|
||||
|
||||
```text
|
||||
notarius vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
when the build has a valid release version, and:
|
||||
|
||||
```text
|
||||
notarius development
|
||||
```
|
||||
|
||||
when no release version is available.
|
||||
|
||||
The implementation must obtain the main-module version from Go build
|
||||
information so `go install ...@vMAJOR.MINOR.PATCH` reports the selected tag. It
|
||||
must also accept an optional link-time version override so controlled builds
|
||||
and release CI can identify an exact tag from a checkout. The override must be
|
||||
validated and must not silently turn arbitrary text into a release version.
|
||||
Ordinary unversioned checkout builds remain `development`; the release process
|
||||
must not modify a tracked source constant for each release.
|
||||
|
||||
Version reporting is an informational product interface. It does not replace
|
||||
receipt, configuration, prompt, or artifact schema versioning, and it must not
|
||||
be used as the sole downstream compatibility check.
|
||||
|
||||
## Release Notes
|
||||
|
||||
Each new `docs/releases/<tag>.md` document has this minimum structure:
|
||||
|
||||
```markdown
|
||||
# Notarius vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
The note should concisely explain the release's purpose, compatibility with the
|
||||
preceding release, operator actions, and material user-visible, operational,
|
||||
integration, and maintainer-visible changes. It should link to canonical
|
||||
current-state documentation for exact contracts rather than duplicating those
|
||||
contracts.
|
||||
|
||||
Release notes are durable historical summaries. They must not contain
|
||||
credentials, private infrastructure detail, sensitive campaign material, or
|
||||
claims that are not true of the tagged candidate. A release note does not
|
||||
excuse stale current-state documentation; affected canonical documents are
|
||||
updated in the same candidate.
|
||||
|
||||
## Candidate Validation
|
||||
|
||||
The release procedure must provide copyable POSIX-shell guards that validate
|
||||
the release version, release-note filename and heading, required note sections,
|
||||
repository state, and module hygiene. Validation must be run from the Notarius
|
||||
repository root with Go workspace behavior disabled.
|
||||
|
||||
At minimum, a candidate must pass:
|
||||
|
||||
- no tracked `go.work` or `go.work.sum`, no vendored tree, and no `replace`
|
||||
directive in `go.mod`;
|
||||
- `GOWORK=off go test -count=1 ./...`;
|
||||
- `GOWORK=off go test -race -count=1 ./...`;
|
||||
- `GOWORK=off go vet ./...`;
|
||||
- `GOWORK=off go build ./...`;
|
||||
- `GOWORK=off go mod tidy -diff`;
|
||||
- `gofmt` verification for every tracked Go file;
|
||||
- `git diff --check` and `git diff --cached --check`;
|
||||
- validation of both maintained D&D configuration examples with their selected
|
||||
pipeline;
|
||||
- Linux `amd64` and `arm64` static command builds;
|
||||
- best-effort Darwin `amd64` and `arm64` static command builds; and
|
||||
- a focused manual or automated check that every added or changed local
|
||||
Markdown link resolves.
|
||||
|
||||
The candidate review also checks for generated binaries, test output,
|
||||
credentials, temporary files, module replacements, vendored dependencies, and
|
||||
other unintended source-control content. Tests remain offline and do not call
|
||||
an LLM provider or require live credentials.
|
||||
|
||||
## Candidate Publication
|
||||
|
||||
The release procedure must guard the exact commit immediately before tagging.
|
||||
It requires:
|
||||
|
||||
- the current branch is `main`;
|
||||
- the worktree and index are clean;
|
||||
- the candidate commit has been pushed and exactly matches `origin/main`;
|
||||
- the matching release note exists in that commit;
|
||||
- no local or remote tag already uses the selected version; and
|
||||
- the substantive release checks have passed for that exact candidate.
|
||||
|
||||
The maintainer records the exact candidate commit, creates a lightweight tag
|
||||
bound explicitly to that commit, verifies the local tag target, and pushes only
|
||||
that tag ref. The procedure must not recommend `git push --tags`.
|
||||
|
||||
After publication, the maintainer verifies that the remote tag resolves to the
|
||||
guarded commit and that the release note can be read from the tagged tree. A
|
||||
fresh temporary checkout or `go install ...@<tag>` must build successfully, and
|
||||
the resulting command must report the expected version through `--version`.
|
||||
|
||||
## Validation-Only Release Automation
|
||||
|
||||
Add a tag-triggered Woodpecker pipeline that validates source releases without
|
||||
publishing artifacts. It should:
|
||||
|
||||
- accept only stable semantic-version tags;
|
||||
- require the version-matched release note;
|
||||
- run the same substantive module, test, race, vet, build, formatting, and
|
||||
whitespace checks as the documented local procedure;
|
||||
- validate the maintained configuration examples;
|
||||
- perform the supported and best-effort cross-build checks; and
|
||||
- verify a release-version build's `notarius --version` output on the CI host.
|
||||
|
||||
The pipeline must not upload binaries, create archives or checksums, create or
|
||||
edit a Gitea release object, or require a release API token. Local guards remain
|
||||
authoritative before tag publication because CI begins only after the tag is
|
||||
already remote.
|
||||
|
||||
If tag validation fails, preserve the published tag, fix the cause on `main`,
|
||||
select a new patch version, and repeat the full process. Do not weaken tag
|
||||
immutability merely because the release contains source rather than binaries.
|
||||
|
||||
## Documentation Ownership
|
||||
|
||||
In the target state:
|
||||
|
||||
- `docs/release.md` owns the maintainer release procedure, commands, ordering,
|
||||
publication checks, and failure recovery;
|
||||
- `docs/releases/` owns one historical summary per release made under the new
|
||||
process;
|
||||
- `docs/cli.md` owns the `--version` contract;
|
||||
- `README.md` owns the shortest source-installation example and links to the
|
||||
release procedure where useful;
|
||||
- `docs/development.md` routes release preparation, tagging, and verification
|
||||
work to `docs/release.md`;
|
||||
- `docs/policy/documentation.md` assigns canonical ownership to the release
|
||||
procedure and release notes;
|
||||
- `docs/policy/architecture.md` records Linux support, best-effort macOS
|
||||
development, unsupported Windows, and source-only distribution only if those
|
||||
are judged durable development invariants rather than release mechanics; and
|
||||
- `docs/operations.md` describes only installation or deployment consequences
|
||||
relevant to operators and links to canonical CLI and release contracts.
|
||||
|
||||
Current-state documentation must not describe the new release process,
|
||||
`--version`, or automated validation until the corresponding behavior exists.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A maintainer can prepare, validate, tag, publish, and verify a source release
|
||||
by following `docs/release.md` without relying on undocumented knowledge.
|
||||
- Every new release has an immutable semantic-version tag and matching
|
||||
checked-in release note in the tagged commit.
|
||||
- The guarded candidate is clean, synchronized with `origin/main`, and passes
|
||||
the documented substantive checks before tagging.
|
||||
- Tag-triggered CI independently validates the published source and never
|
||||
publishes binary artifacts.
|
||||
- `go install` of a tagged version succeeds and `notarius --version` reports
|
||||
that version; ordinary unversioned builds report `development`.
|
||||
- Linux is the documented supported deployment platform, macOS has a
|
||||
best-effort development build check, and Windows is explicitly unsupported.
|
||||
- Downstream compatibility remains based on durable Notarius contracts rather
|
||||
than the product version alone.
|
||||
- Existing pre-procedure tags remain untouched and require no invented release
|
||||
history.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Publishing executable archives, installers, container images, checksums,
|
||||
signatures, or package-manager entries.
|
||||
- Supporting or cross-compiling for Windows.
|
||||
- Creating or maintaining a mutable Gitea release page.
|
||||
- Supporting prerelease tag syntax in the initial procedure.
|
||||
- Automating version selection, release-note authorship, commits, or tag
|
||||
creation.
|
||||
- Retrospectively creating release notes for `v0.1.0` through `v0.3.0`.
|
||||
- Treating a product version as a substitute for receipt, configuration,
|
||||
prompt, or artifact schema compatibility.
|
||||
2
go.mod
2
go.mod
@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@@ -1,5 +1,5 @@
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0 h1:NGd9hDLu0UMxKbvittMrqM5Ua94eFb+kOE7UIir8l08=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
|
||||
36
internal/buildinfo/buildinfo.go
Normal file
36
internal/buildinfo/buildinfo.go
Normal file
@@ -0,0 +1,36 @@
|
||||
// Package buildinfo resolves the product version embedded in a Notarius build.
|
||||
package buildinfo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
|
||||
|
||||
// Override is set at link time for controlled builds.
|
||||
var Override string
|
||||
|
||||
// Version returns the release version embedded in the build, or development
|
||||
// when the build does not carry a stable release tag.
|
||||
func Version() (string, error) {
|
||||
buildVersion := ""
|
||||
if info, ok := debug.ReadBuildInfo(); ok {
|
||||
buildVersion = info.Main.Version
|
||||
}
|
||||
return resolve(Override, buildVersion)
|
||||
}
|
||||
|
||||
func resolve(override, buildVersion string) (string, error) {
|
||||
if override != "" {
|
||||
if !stableVersion.MatchString(override) {
|
||||
return "", fmt.Errorf("build version override is not a stable release tag")
|
||||
}
|
||||
return override, nil
|
||||
}
|
||||
if stableVersion.MatchString(buildVersion) {
|
||||
return buildVersion, nil
|
||||
}
|
||||
return "development", nil
|
||||
}
|
||||
45
internal/buildinfo/buildinfo_test.go
Normal file
45
internal/buildinfo/buildinfo_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package buildinfo
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestResolve(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
override string
|
||||
buildVersion string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
|
||||
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
|
||||
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
|
||||
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
|
||||
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
|
||||
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
|
||||
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
|
||||
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
|
||||
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
|
||||
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
|
||||
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
|
||||
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
|
||||
{name: "development build", buildVersion: "(devel)", want: "development"},
|
||||
{name: "missing build information", want: "development"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := resolve(tt.override, tt.buildVersion)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("resolve() error = nil, want error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("resolve() error = %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("resolve() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||
)
|
||||
|
||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandVersionOutput(t *testing.T) {
|
||||
previous := buildinfo.Override
|
||||
t.Cleanup(func() { buildinfo.Override = previous })
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
override string
|
||||
wantCode int
|
||||
wantStdout string
|
||||
wantStderr string
|
||||
}{
|
||||
{name: "development", wantStdout: "notarius development\n"},
|
||||
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
|
||||
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
buildinfo.Override = tt.override
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
|
||||
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||
environment := writeCommandConfig(t, "environment", "beta")
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||
@@ -331,27 +333,53 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("production prompt engine: %v", err)
|
||||
}
|
||||
inputs := map[string]promptkit.ArtifactRef{
|
||||
"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Alias","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
|
||||
promptFS, err := components.assets.PromptFS()
|
||||
if err != nil {
|
||||
t.Fatalf("production prompt assets: %v", err)
|
||||
}
|
||||
for _, prompt := range []struct {
|
||||
id string
|
||||
version string
|
||||
}{
|
||||
{id: npcnormalize.PromptID, version: npcnormalize.PromptVersion},
|
||||
{id: itemregistrynormalize.PromptID, version: itemregistrynormalize.PromptVersion},
|
||||
{id: locationnormalize.PromptID, version: locationnormalize.PromptVersion},
|
||||
} {
|
||||
type manifest struct {
|
||||
ID string `yaml:"id"`
|
||||
Version string `yaml:"version"`
|
||||
Inputs []struct {
|
||||
Name string `yaml:"name"`
|
||||
} `yaml:"inputs"`
|
||||
}
|
||||
preparedPrompts := 0
|
||||
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
|
||||
return nil
|
||||
}
|
||||
data, err := fs.ReadFile(promptFS, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var prompt manifest
|
||||
if err := yaml.Unmarshal(data, &prompt); err != nil {
|
||||
return err
|
||||
}
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
|
||||
for _, input := range prompt.Inputs {
|
||||
inputs[input.Name] = promptkit.Inline(`{}`)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: prompt.id, PromptVersion: prompt.version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
||||
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare production prompt %q: %v", prompt.id, err)
|
||||
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != filepath.Base(semanticreconcile.SchemaAssetPath) {
|
||||
t.Fatalf("prompt %q schema = %q, want generic reconciliation schema", prompt.id, prepared.OutputContract.SchemaPath)
|
||||
if prepared.OutputContract.RepairAttempts != 1 {
|
||||
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
|
||||
}
|
||||
preparedPrompts++
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if preparedPrompts == 0 {
|
||||
t.Fatal("prepared no production prompts")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
},
|
||||
{
|
||||
name: "malformed profile",
|
||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
|
||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||
profileID: "malformed-profile",
|
||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
|
||||
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||
},
|
||||
{
|
||||
name: "invalid profile source",
|
||||
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||
for _, profiles := range []string{
|
||||
"id: child\nbase_profile: missing\n",
|
||||
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||
} {
|
||||
t.Run("invalid inheritance", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profileID := "child"
|
||||
if strings.Contains(profiles, "id: first") {
|
||||
profileID = "first"
|
||||
}
|
||||
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
|
||||
t.Fatalf("profile preflight error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
@@ -31,6 +32,7 @@ import (
|
||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
||||
const usage = `Usage:
|
||||
notarius help
|
||||
notarius --version
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list --config path/to/config.yml [--json]
|
||||
@@ -62,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
|
||||
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
if len(args) > 0 && args[0] == "--version" {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
|
||||
return 2
|
||||
}
|
||||
version, err := buildinfo.Version()
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(stdout, "notarius %s\n", version)
|
||||
return 0
|
||||
}
|
||||
|
||||
var err error
|
||||
opts, err = normalizeOptions(opts)
|
||||
if err != nil {
|
||||
|
||||
@@ -48,8 +48,15 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != want {
|
||||
t.Fatalf("%s mode = %#o, want %#o", path, info.Mode().Perm(), want)
|
||||
mode := info.Mode().Perm()
|
||||
if mode&^want != 0 {
|
||||
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
|
||||
}
|
||||
if info.IsDir() && mode&0o700 != 0o700 {
|
||||
t.Fatalf("%s mode = %#o, want owner access", path, mode)
|
||||
}
|
||||
if !info.IsDir() && mode != want {
|
||||
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
||||
|
||||
@@ -116,6 +116,10 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
|
||||
|
||||
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||
out := in
|
||||
if in.StructuredOutputRepairAttempts != nil {
|
||||
value := *in.StructuredOutputRepairAttempts
|
||||
out.StructuredOutputRepairAttempts = &value
|
||||
}
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
@@ -196,6 +200,10 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
|
||||
|
||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||
out := in
|
||||
if in.StructuredOutputRepairAttempts != nil {
|
||||
value := *in.StructuredOutputRepairAttempts
|
||||
out.StructuredOutputRepairAttempts = &value
|
||||
}
|
||||
if len(in.Options) > 0 {
|
||||
out.Options = cloneOptions(in.Options)
|
||||
}
|
||||
|
||||
@@ -35,23 +35,27 @@ type FilePromptKitLocalBackendConfig struct {
|
||||
}
|
||||
|
||||
type FilePipelineProfile struct {
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||
artifactsSet bool `yaml:"-"`
|
||||
stepsSet bool `yaml:"-"`
|
||||
llmProfileSet bool `yaml:"-"`
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||
artifactsSet bool `yaml:"-"`
|
||||
stepsSet bool `yaml:"-"`
|
||||
llmProfileSet bool `yaml:"-"`
|
||||
}
|
||||
|
||||
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
|
||||
return err
|
||||
}
|
||||
type plainFilePipelineProfile FilePipelineProfile
|
||||
var decoded plainFilePipelineProfile
|
||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||
}, "pipeline profile")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -147,12 +151,13 @@ type FileDebugConfig struct {
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
Retries int
|
||||
Options map[string]any
|
||||
References map[string]fileReferenceSource
|
||||
Validators pipeline.ValidatorOverride
|
||||
Module string
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Retries int
|
||||
Options map[string]any
|
||||
References map[string]fileReferenceSource
|
||||
Validators pipeline.ValidatorOverride
|
||||
}
|
||||
|
||||
type fileReferenceSource struct {
|
||||
@@ -264,6 +269,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
if b.LLMProfile == "" {
|
||||
return fmt.Errorf("llm_profile must not be empty when set")
|
||||
}
|
||||
case "structured_output_repair_attempts":
|
||||
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.StructuredOutputRepairAttempts = attempts
|
||||
case "retries":
|
||||
var retries int
|
||||
if err := valueNode.Decode(&retries); err != nil {
|
||||
@@ -304,15 +315,56 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
|
||||
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
return pipeline.ModuleBinding{
|
||||
Module: strings.TrimSpace(b.Module),
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
Retries: b.Retries,
|
||||
Options: cloneOptions(b.Options),
|
||||
References: fileReferenceSourcesToPipeline(b.References),
|
||||
Validators: b.Validators,
|
||||
Module: strings.TrimSpace(b.Module),
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
|
||||
Retries: b.Retries,
|
||||
Options: cloneOptions(b.Options),
|
||||
References: fileReferenceSourcesToPipeline(b.References),
|
||||
Validators: cloneValidatorOverride(b.Validators),
|
||||
}
|
||||
}
|
||||
|
||||
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("%s must be an object", context)
|
||||
}
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value != "structured_output_repair_attempts" {
|
||||
continue
|
||||
}
|
||||
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
|
||||
if node.Tag == "!!null" {
|
||||
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
|
||||
}
|
||||
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
|
||||
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
|
||||
}
|
||||
var attempts int
|
||||
if err := node.Decode(&attempts); err != nil {
|
||||
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
|
||||
}
|
||||
if attempts < 0 || attempts > 3 {
|
||||
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||
}
|
||||
return &attempts, nil
|
||||
}
|
||||
|
||||
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
|
||||
if attempts == nil {
|
||||
return nil
|
||||
}
|
||||
value := *attempts
|
||||
return &value
|
||||
}
|
||||
|
||||
func LoadFileConfig(path string) (FileConfig, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -518,11 +570,12 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
return err
|
||||
}
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: pipelineID,
|
||||
LLMProfile: llmProfile,
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||
ID: pipelineID,
|
||||
LLMProfile: llmProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
||||
|
||||
@@ -138,6 +138,168 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
|
||||
const pipelineYAML = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
%s
|
||||
input: input
|
||||
artifacts:
|
||||
lane:
|
||||
extract:
|
||||
module: extract
|
||||
structured_output_repair_attempts: 2
|
||||
validators:
|
||||
- module: validator
|
||||
structured_output_repair_attempts: 3
|
||||
`
|
||||
|
||||
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
|
||||
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
|
||||
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
|
||||
}
|
||||
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
|
||||
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
|
||||
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := cfg.Pipelines["main"]
|
||||
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
|
||||
}
|
||||
lane := profile.Artifacts["lane"]
|
||||
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
|
||||
}
|
||||
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||
}
|
||||
|
||||
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
|
||||
}
|
||||
fileProfile := file.Pipelines["main"]
|
||||
fileLane := fileProfile.Artifacts["lane"]
|
||||
*fileLane.Extract.StructuredOutputRepairAttempts = 1
|
||||
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
|
||||
fileProfile.Artifacts["lane"] = fileLane
|
||||
file.Pipelines["main"] = fileProfile
|
||||
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
|
||||
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
|
||||
}
|
||||
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
|
||||
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
|
||||
}
|
||||
cloned := cloneConfig(cfg)
|
||||
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
|
||||
}
|
||||
redacted := cfg.Redacted()
|
||||
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
|
||||
}
|
||||
summary := cfg.RedactedSummaryPayload().(Config)
|
||||
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var roundTripped Config
|
||||
if err := json.Unmarshal(data, &roundTripped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roundTrippedProfile := roundTripped.Pipelines["main"]
|
||||
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
|
||||
}
|
||||
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
|
||||
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
|
||||
}
|
||||
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
|
||||
const pipelineYAML = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input: input
|
||||
artifacts:
|
||||
lane:
|
||||
extract: extract
|
||||
%s
|
||||
`
|
||||
const bindingYAML = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input:
|
||||
module: input
|
||||
%s
|
||||
artifacts:
|
||||
lane:
|
||||
extract: extract
|
||||
`
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
source string
|
||||
want string
|
||||
}{
|
||||
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
|
||||
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
source string
|
||||
want string
|
||||
}{
|
||||
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
|
||||
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
|
||||
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
||||
const configYAML = `version: 4
|
||||
pipelines:
|
||||
|
||||
@@ -116,6 +116,9 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||
}
|
||||
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -195,6 +198,9 @@ func validateBinding(
|
||||
binding pipeline.ModuleBinding,
|
||||
referencesAllowed bool,
|
||||
) error {
|
||||
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -219,6 +225,13 @@ func validateBinding(
|
||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
|
||||
}
|
||||
|
||||
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
|
||||
if attempts != nil && (*attempts < 0 || *attempts > 3) {
|
||||
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||
if !override.Set {
|
||||
return nil
|
||||
@@ -230,6 +243,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
|
||||
}
|
||||
for i, validator := range override.Validators {
|
||||
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(validator.Module) == "" {
|
||||
return fmt.Errorf("%s module must not be empty", context)
|
||||
}
|
||||
|
||||
@@ -9,13 +9,14 @@ import (
|
||||
)
|
||||
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
}
|
||||
|
||||
type StructuredCompletionResponse struct {
|
||||
@@ -26,6 +27,7 @@ type StructuredCompletionResponse struct {
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
||||
}
|
||||
|
||||
@@ -60,19 +62,27 @@ type LLMDebugMessage struct {
|
||||
}
|
||||
|
||||
type LLMDebugResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Validation map[string]any `json:"validation,omitempty"`
|
||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Validation map[string]any `json:"validation,omitempty"`
|
||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||
ProviderError *LLMDebugProviderError `json:"provider_error,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugProviderError struct {
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugUsage struct {
|
||||
@@ -126,11 +136,12 @@ func (set LLMInputSet) Clone() LLMInputSet {
|
||||
}
|
||||
|
||||
type ParseRequest struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw []byte `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw []byte `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type InputAdapter interface {
|
||||
@@ -139,12 +150,13 @@ type InputAdapter interface {
|
||||
}
|
||||
|
||||
type ChunkRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkPlanResult struct {
|
||||
@@ -282,14 +294,15 @@ type Warning struct {
|
||||
}
|
||||
|
||||
type OutputRequest struct {
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
|
||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
||||
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
|
||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
||||
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
||||
}
|
||||
|
||||
type OutputFile struct {
|
||||
|
||||
@@ -9,3 +9,31 @@ var ErrInvalidStructuredOutput = errors.New("invalid structured output")
|
||||
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
|
||||
// generation begins.
|
||||
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
|
||||
|
||||
// ErrLLMGeneration identifies a provider generation failure.
|
||||
var ErrLLMGeneration = errors.New("LLM generation failed")
|
||||
|
||||
type LLMGenerationError struct {
|
||||
status int
|
||||
diagnostic string
|
||||
}
|
||||
|
||||
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
|
||||
if status < 0 {
|
||||
status = 0
|
||||
}
|
||||
return &LLMGenerationError{status: status, diagnostic: diagnostic}
|
||||
}
|
||||
func (e *LLMGenerationError) Error() string {
|
||||
if e == nil || e.diagnostic == "" {
|
||||
return ErrLLMGeneration.Error()
|
||||
}
|
||||
return e.diagnostic
|
||||
}
|
||||
func (e *LLMGenerationError) Unwrap() error { return ErrLLMGeneration }
|
||||
func (e *LLMGenerationError) StatusCode() int {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.status
|
||||
}
|
||||
|
||||
@@ -34,14 +34,15 @@ type NormalizeArtifact[T any] struct {
|
||||
}
|
||||
|
||||
type TypedExtractionRequest struct {
|
||||
Source *source.SourceDocument
|
||||
Chunk *source.Chunk
|
||||
AmbientContext map[string]any
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Source *source.SourceDocument
|
||||
Chunk *source.Chunk
|
||||
AmbientContext map[string]any
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedExtractionResult[T any] struct {
|
||||
@@ -56,14 +57,15 @@ type Extractor[T any] interface {
|
||||
}
|
||||
|
||||
type TypedMergeRequest[T any] struct {
|
||||
Source *source.SourceDocument
|
||||
LaneID string
|
||||
ExtractOutputs []ExtractArtifact[T]
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Source *source.SourceDocument
|
||||
LaneID string
|
||||
ExtractOutputs []ExtractArtifact[T]
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedMergeResult[T any] struct {
|
||||
@@ -77,14 +79,15 @@ type Merger[T any] interface {
|
||||
}
|
||||
|
||||
type TypedNormalizeRequest[T any] struct {
|
||||
Source *source.SourceDocument
|
||||
LaneID string
|
||||
MergeOutput MergeArtifact[T]
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Source *source.SourceDocument
|
||||
LaneID string
|
||||
MergeOutput MergeArtifact[T]
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedNormalizeResult[T any] struct {
|
||||
@@ -115,20 +118,21 @@ type Normalizer[T any] interface {
|
||||
}
|
||||
|
||||
type TypedValidationRequest[T any] struct {
|
||||
Stage string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Chunk *source.Chunk
|
||||
Chunks []source.Chunk
|
||||
Ref source.SourceRef
|
||||
Value T
|
||||
Stage string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
Chunk *source.Chunk
|
||||
Chunks []source.Chunk
|
||||
Ref source.SourceRef
|
||||
Value T
|
||||
}
|
||||
|
||||
type TypedValidator[T any] interface {
|
||||
@@ -138,15 +142,16 @@ type TypedValidator[T any] interface {
|
||||
}
|
||||
|
||||
type ChunkValidationRequest struct {
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Chunks []source.Chunk
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
Chunks []source.Chunk
|
||||
}
|
||||
|
||||
type ChunkValidator interface {
|
||||
@@ -156,21 +161,22 @@ type ChunkValidator interface {
|
||||
}
|
||||
|
||||
type SerializedValidationRequest struct {
|
||||
Stage string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
Metadata map[string]any
|
||||
Chunk *source.Chunk
|
||||
Chunks []source.Chunk
|
||||
Schema ArtifactSchema
|
||||
MediaType string
|
||||
Content []byte
|
||||
Stage string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
Source *source.SourceDocument
|
||||
SourceID string
|
||||
SourceInput LLMInputMaterial
|
||||
SessionID string
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Metadata map[string]any
|
||||
Chunk *source.Chunk
|
||||
Chunks []source.Chunk
|
||||
Schema ArtifactSchema
|
||||
MediaType string
|
||||
Content []byte
|
||||
}
|
||||
|
||||
type SerializedValidator interface {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -115,6 +116,9 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
|
||||
}
|
||||
promptID := strings.TrimSpace(req.PromptID)
|
||||
if promptID == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||
@@ -137,6 +141,18 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
Vars: promptKitVars(req, sessionID),
|
||||
Execution: execution,
|
||||
}
|
||||
if req.StructuredOutputRepairAttempts != nil {
|
||||
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("inspect PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||
}
|
||||
contract := inspection.OutputContract
|
||||
contract.RepairAttempts = *req.StructuredOutputRepairAttempts
|
||||
runReq.Validation = &contract
|
||||
}
|
||||
prepared, err := c.engine.PrepareExecution(ctx, runReq)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
@@ -169,6 +185,16 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
redactPromptKitError(err),
|
||||
)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
var generationErr *promptkit.GenerationError
|
||||
status := 0
|
||||
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
|
||||
if errors.As(err, &generationErr) {
|
||||
status = generationErr.StatusCode()
|
||||
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
|
||||
}
|
||||
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||
}
|
||||
if result == nil {
|
||||
@@ -187,6 +213,27 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
|
||||
if generationErr == nil {
|
||||
return nil
|
||||
}
|
||||
var secrets []string
|
||||
if prepared != nil && strings.TrimSpace(prepared.EffectiveModelParams.APIKeyEnv) != "" {
|
||||
if value, ok := os.LookupEnv(prepared.EffectiveModelParams.APIKeyEnv); ok {
|
||||
secrets = append(secrets, value)
|
||||
}
|
||||
}
|
||||
redact := func(value string) string {
|
||||
return bearerTokenPattern.ReplaceAllString(RedactSecrets(value, secrets), "Bearer "+secretReplacement)
|
||||
}
|
||||
return &contracts.LLMDebugResponse{ProviderError: &contracts.LLMDebugProviderError{
|
||||
StatusCode: generationErr.StatusCode(),
|
||||
Code: redact(generationErr.ProviderCode()),
|
||||
Type: redact(generationErr.ProviderType()),
|
||||
Message: redact(generationErr.ProviderMessage()),
|
||||
}}
|
||||
}
|
||||
|
||||
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
||||
content := result.Artifact.Body
|
||||
if len(content) == 0 {
|
||||
@@ -210,6 +257,7 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
|
||||
PromptTokens: result.Usage.PromptTokens,
|
||||
CompletionTokens: result.Usage.CompletionTokens,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
RepairAttempts: result.Validation.RepairAttempts,
|
||||
Debug: promptKitDebugMaterial(prepared, result),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
@@ -456,6 +457,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inherited parent", func(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||
@@ -531,6 +558,130 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
|
||||
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
|
||||
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
|
||||
t.Fatalf("inspection = %#v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "" {
|
||||
t.Fatalf("Authorization = %q, want omitted", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
||||
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.T) {
|
||||
const credential = "selected-test-credential"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":"temporary","type":"provider_error","message":"marker ` + credential + ` Bearer bearer-secret"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
|
||||
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "generation-profile", SessionID: "generation-error-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
var generation *contracts.LLMGenerationError
|
||||
if !errors.As(err, &generation) || generation.StatusCode() != http.StatusServiceUnavailable {
|
||||
t.Fatalf("generation error = %#v", generation)
|
||||
}
|
||||
if strings.Contains(err.Error(), credential) || strings.Contains(err.Error(), "marker") {
|
||||
t.Fatalf("ordinary error leaked provider detail: %v", err)
|
||||
}
|
||||
if response.Debug == nil || response.Debug.Response == nil || response.Debug.Response.ProviderError == nil {
|
||||
t.Fatalf("debug = %#v", response.Debug)
|
||||
}
|
||||
debug := response.Debug.Response.ProviderError
|
||||
if debug.StatusCode != http.StatusServiceUnavailable || debug.Code != "temporary" || strings.Contains(debug.Message, credential) || strings.Contains(debug.Message, "bearer-secret") || !strings.Contains(debug.Message, "marker") {
|
||||
t.Fatalf("debug provider error = %#v", debug)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientRejectsInvalidRepairAttemptOverride(t *testing.T) {
|
||||
for _, attempts := range []int{-1, 4} {
|
||||
t.Run("invalid", func(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", StructuredOutputRepairAttempts: &attempts}, &out)
|
||||
if err == nil || fake.calls != 0 {
|
||||
t.Fatalf("error=%v calls=%d", err, fake.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
@@ -772,12 +923,15 @@ func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
|
||||
attempts := 1
|
||||
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{{Content: `{"bad":true}`}, {Content: `{"bad":true}`}}}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
|
||||
var out map[string]any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
@@ -785,6 +939,9 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
||||
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||
t.Fatalf("provider calls = %d, want exhausted repair budget", got)
|
||||
}
|
||||
if got := string(resp.Content); got != `{"bad":true}` {
|
||||
t.Fatalf("response content = %q, want raw failed output", got)
|
||||
}
|
||||
@@ -796,6 +953,45 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *testing.T) {
|
||||
attempts := 1
|
||||
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{
|
||||
{Content: `{"bad":true}`, Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8}},
|
||||
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
|
||||
}}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
|
||||
var out struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
SessionID: "repair-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err != nil || !out.OK {
|
||||
t.Fatalf("CompleteStructured() = (%#v, %v), want repaired success", response, err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
|
||||
}
|
||||
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
|
||||
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
|
||||
}
|
||||
if response.Debug == nil || response.Debug.Prompt == nil || response.Debug.Response == nil {
|
||||
t.Fatalf("debug = %#v, want prompt and response details", response.Debug)
|
||||
}
|
||||
if response.Debug.Prompt.OutputContract["repair_attempts"] != float64(1) ||
|
||||
response.Debug.Response.Validation["repair_attempts"] != float64(1) ||
|
||||
response.Debug.Response.Content != `{"ok":true}` ||
|
||||
response.Debug.Response.Usage.TotalTokens != 26 {
|
||||
t.Fatalf("debug repair result = %#v, want configured contract and repaired response", response.Debug)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
|
||||
@@ -847,8 +1043,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
||||
t.Fatalf("error chain exposes credential: %v", err)
|
||||
}
|
||||
}
|
||||
if errors.Unwrap(err) != nil {
|
||||
t.Fatalf("provider failure must not expose a wrapped diagnostic: %v", err)
|
||||
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||
t.Fatalf("provider failure = %v, want application generation classification", err)
|
||||
}
|
||||
var recoveredProviderErr *credentialBearingProviderError
|
||||
if errors.As(err, &recoveredProviderErr) {
|
||||
@@ -857,8 +1053,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
||||
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
|
||||
}
|
||||
if resp.Debug != nil {
|
||||
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
|
||||
if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Response != nil {
|
||||
t.Fatalf("debug material = %#v, want prepared prompt without provider details", resp.Debug)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -951,19 +1147,31 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
attempts := 1
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
attempts *int
|
||||
}{
|
||||
{name: "during preparation"},
|
||||
{name: "during override inspection", attempts: &attempts},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
StructuredOutputRepairAttempts: test.attempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1071,6 +1279,66 @@ func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
|
||||
attempts := 1
|
||||
fake := &fakePromptKitLLM{
|
||||
responses: []promptkit.GenerateResponse{
|
||||
{Content: `{"bad":true}`},
|
||||
{Content: `{"ok":true}`},
|
||||
{Content: `{"ok":true}`},
|
||||
},
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v", err)
|
||||
}
|
||||
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
|
||||
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "repair-session",
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
firstDone <- callErr
|
||||
}()
|
||||
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||
|
||||
secondDone := make(chan error, 1)
|
||||
go func() {
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "queued-session",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
secondDone <- callErr
|
||||
}()
|
||||
close(fake.block)
|
||||
if err := <-firstDone; err != nil {
|
||||
t.Fatalf("repaired completion error = %v", err)
|
||||
}
|
||||
if err := <-secondDone; err != nil {
|
||||
t.Fatalf("queued completion error = %v", err)
|
||||
}
|
||||
|
||||
requests := fake.requestsSnapshot()
|
||||
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
|
||||
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
var out map[string]any
|
||||
@@ -1225,11 +1493,13 @@ func (f *switchingPromptFS) resumePromptRead() {
|
||||
|
||||
type fakePromptKitLLM struct {
|
||||
content string
|
||||
responses []promptkit.GenerateResponse
|
||||
allowEmpty bool
|
||||
err error
|
||||
block chan struct{}
|
||||
mu sync.Mutex
|
||||
last promptkit.GenerateRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
calls int32
|
||||
inFlight int32
|
||||
maxInFlight int32
|
||||
@@ -1244,8 +1514,9 @@ func (*credentialBearingProviderError) Error() string {
|
||||
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
f.mu.Lock()
|
||||
f.last = req
|
||||
f.requests = append(f.requests, req)
|
||||
f.mu.Unlock()
|
||||
atomic.AddInt32(&f.calls, 1)
|
||||
call := atomic.AddInt32(&f.calls, 1)
|
||||
current := atomic.AddInt32(&f.inFlight, 1)
|
||||
for {
|
||||
seen := atomic.LoadInt32(&f.maxInFlight)
|
||||
@@ -1264,6 +1535,14 @@ func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateR
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if len(f.responses) > 0 {
|
||||
index := int(call - 1)
|
||||
if index >= len(f.responses) {
|
||||
return nil, fmt.Errorf("unexpected provider call %d", call)
|
||||
}
|
||||
response := f.responses[index]
|
||||
return &response, nil
|
||||
}
|
||||
content := f.content
|
||||
if content == "" && !f.allowEmpty {
|
||||
content = `{"ok":true}`
|
||||
@@ -1294,3 +1573,9 @@ func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
|
||||
defer f.mu.Unlock()
|
||||
return f.last
|
||||
}
|
||||
|
||||
func (f *fakePromptKitLLM) requestsSnapshot() []promptkit.GenerateRequest {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]promptkit.GenerateRequest(nil), f.requests...)
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ const (
|
||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||
// release. Update this identity when the dependency is upgraded.
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.8.0:builtin-profiles"
|
||||
)
|
||||
|
||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||
|
||||
@@ -85,7 +85,7 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
||||
}
|
||||
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
||||
}
|
||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Metadata: request.Metadata})
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Metadata: request.Metadata})
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
|
||||
@@ -21,12 +21,13 @@ const (
|
||||
)
|
||||
|
||||
type ModuleBinding struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactReference identifies a normalized artifact produced by an earlier
|
||||
@@ -80,19 +81,21 @@ type ValidatorOverride struct {
|
||||
|
||||
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
type moduleBindingJSON struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||
}
|
||||
out := moduleBindingJSON{
|
||||
Module: binding.Module,
|
||||
LLMProfile: binding.LLMProfile,
|
||||
Retries: binding.Retries,
|
||||
Options: binding.Options,
|
||||
References: binding.References,
|
||||
Module: binding.Module,
|
||||
LLMProfile: binding.LLMProfile,
|
||||
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
|
||||
Retries: binding.Retries,
|
||||
Options: binding.Options,
|
||||
References: binding.References,
|
||||
}
|
||||
if binding.Validators.Set {
|
||||
validators := cloneModuleBindings(binding.Validators.Validators)
|
||||
@@ -101,6 +104,42 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||
if binding == nil {
|
||||
return fmt.Errorf("module binding must not be nil")
|
||||
}
|
||||
type moduleBindingJSON struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
Validators json.RawMessage `json:"validators,omitempty"`
|
||||
}
|
||||
var decoded moduleBindingJSON
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
return err
|
||||
}
|
||||
out := ModuleBinding{
|
||||
Module: decoded.Module,
|
||||
LLMProfile: decoded.LLMProfile,
|
||||
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
|
||||
Retries: decoded.Retries,
|
||||
Options: decoded.Options,
|
||||
References: decoded.References,
|
||||
}
|
||||
if len(decoded.Validators) > 0 && string(decoded.Validators) != "null" {
|
||||
var validators []ModuleBinding
|
||||
if err := json.Unmarshal(decoded.Validators, &validators); err != nil {
|
||||
return fmt.Errorf("decode module binding validators: %w", err)
|
||||
}
|
||||
out.Validators = ValidatorOverride{Set: true, Validators: validators}
|
||||
}
|
||||
*binding = cloneModuleBinding(out)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ArtifactLaneProfile struct {
|
||||
Extract ModuleBinding `json:"extract"`
|
||||
Merge ModuleBinding `json:"merge,omitempty"`
|
||||
@@ -116,14 +155,15 @@ type PipelineStepProfile struct {
|
||||
}
|
||||
|
||||
type PipelineProfile struct {
|
||||
ID string `json:"id"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||
Steps []PipelineStepProfile `json:"steps,omitempty"`
|
||||
Output ModuleBinding `json:"output,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
ID string `json:"id"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||
Steps []PipelineStepProfile `json:"steps,omitempty"`
|
||||
Output ModuleBinding `json:"output,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
}
|
||||
|
||||
type ResolveOptions struct {
|
||||
@@ -450,6 +490,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
@@ -1282,12 +1325,13 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
|
||||
return ModuleBinding{}, err
|
||||
}
|
||||
return ModuleBinding{
|
||||
Module: module,
|
||||
LLMProfile: llmProfile,
|
||||
Retries: binding.Retries,
|
||||
Options: cloneOptions(binding.Options),
|
||||
References: references,
|
||||
Validators: cloneValidatorOverride(binding.Validators),
|
||||
Module: module,
|
||||
LLMProfile: llmProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
|
||||
Retries: binding.Retries,
|
||||
Options: cloneOptions(binding.Options),
|
||||
References: references,
|
||||
Validators: cloneValidatorOverride(binding.Validators),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1359,6 +1403,59 @@ func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, over
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pipelineAttempts *int) error {
|
||||
apply := func(stage ModuleStage, laneID, module string, binding *ModuleBinding, executionClass contracts.ExecutionClass, kind string) error {
|
||||
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||
if executionClass != contracts.ExecutionClassLLMBacked {
|
||||
if binding.StructuredOutputRepairAttempts != nil {
|
||||
if laneID == "" {
|
||||
return fmt.Errorf("pipeline %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, stage, module, kind, binding.Module)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q lane %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, laneID, stage, module, kind, binding.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if binding.StructuredOutputRepairAttempts == nil {
|
||||
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(pipelineAttempts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := apply(StageInput, "", resolved.Input.Module, &resolved.Input, resolved.InputExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := apply(StageChunk, "", resolved.Chunk.Module, &resolved.Chunk, resolved.ChunkExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
for stepIndex := range resolved.Steps {
|
||||
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||
if err := apply(StageExtract, lane.ID, lane.Extract.Module, &lane.Extract, lane.ExtractExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := apply(StageMerge, lane.ID, lane.Merge.Module, &lane.Merge, lane.MergeExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := apply(StageNormalize, lane.ID, lane.Normalize.Module, &lane.Normalize, lane.NormalizeExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := apply(StageOutput, "", resolved.Output.Module, &resolved.Output, resolved.OutputExecutionClass, "module"); err != nil {
|
||||
return err
|
||||
}
|
||||
for chainIndex := range resolved.ValidatorChains {
|
||||
chain := &resolved.ValidatorChains[chainIndex]
|
||||
for validatorIndex := range chain.Validators {
|
||||
validator := &chain.Validators[validatorIndex]
|
||||
if err := apply(chain.Stage, chain.LaneID, chain.ModuleKey, &validator.Binding, validator.ExecutionClass, "validator"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
||||
if len(bindings) == 0 {
|
||||
return nil, nil
|
||||
|
||||
@@ -408,6 +408,148 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
|
||||
pipelineAttempts := 2
|
||||
chunkAttempts := 1
|
||||
extractAttempts := 0
|
||||
validatorAttempts := 3
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = &pipelineAttempts
|
||||
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
encoded, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(resolved) error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
|
||||
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
|
||||
}
|
||||
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
got *int
|
||||
want int
|
||||
}{
|
||||
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
|
||||
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
|
||||
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.got == nil || *test.got != test.want {
|
||||
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
|
||||
if extractChain == nil || len(extractChain.Validators) != 1 {
|
||||
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
|
||||
}
|
||||
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
|
||||
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
|
||||
}
|
||||
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
|
||||
if chunkChain == nil || len(chunkChain.Validators) != 1 {
|
||||
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
|
||||
}
|
||||
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
|
||||
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
|
||||
}
|
||||
|
||||
pipelineAttempts = 1
|
||||
chunkAttempts = 2
|
||||
extractAttempts = 3
|
||||
validatorAttempts = 0
|
||||
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
|
||||
}
|
||||
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
|
||||
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
|
||||
}
|
||||
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
|
||||
}
|
||||
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
|
||||
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*PipelineProfile)
|
||||
}{
|
||||
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
|
||||
{name: "validator", mutate: func(profile *PipelineProfile) {
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
test.mutate(&profile)
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.StructuredOutputRepairAttempts = repairAttempts(2)
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
|
||||
}
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
|
||||
digests := make(map[string]string)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
attempts *int
|
||||
}{
|
||||
{name: "prompt owned", attempts: nil},
|
||||
{name: "disabled", attempts: repairAttempts(0)},
|
||||
{name: "configured", attempts: repairAttempts(1)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = test.attempts
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
digests[test.name] = resolved.Digest
|
||||
})
|
||||
}
|
||||
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
|
||||
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
@@ -1843,6 +1985,10 @@ func llmProfilePipeline() PipelineProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func repairAttempts(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
func llmProfileValues(profile string) map[string]string {
|
||||
return map[string]string{
|
||||
"input": profile,
|
||||
|
||||
@@ -173,11 +173,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), err
|
||||
}
|
||||
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.pipeline.Input.LLMProfile,
|
||||
Metadata: requestMetadata,
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.pipeline.Input.LLMProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Input.StructuredOutputRepairAttempts),
|
||||
Metadata: requestMetadata,
|
||||
})
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), ctxErr)
|
||||
@@ -360,14 +361,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), err
|
||||
}
|
||||
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||
Manifest: output.Manifest,
|
||||
NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
|
||||
Rejected: cloneRejectedOutputs(output.Rejected),
|
||||
Warnings: output.Warnings,
|
||||
LLMProfile: input.pipeline.Output.LLMProfile,
|
||||
Metadata: outputMetadata,
|
||||
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
||||
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
||||
Manifest: output.Manifest,
|
||||
NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
|
||||
Rejected: cloneRejectedOutputs(output.Rejected),
|
||||
Warnings: output.Warnings,
|
||||
LLMProfile: input.pipeline.Output.LLMProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Output.StructuredOutputRepairAttempts),
|
||||
Metadata: outputMetadata,
|
||||
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
||||
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
||||
})
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return failOutput(output), ctxErr
|
||||
@@ -491,9 +493,9 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
||||
}
|
||||
switch item.resolved.Target {
|
||||
case ValidatorTargetChunk:
|
||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks})
|
||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
|
||||
case ValidatorTargetSerialized:
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
|
||||
}
|
||||
|
||||
@@ -117,6 +117,38 @@ func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
|
||||
return prepared
|
||||
}
|
||||
|
||||
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
producerAttempts := 2
|
||||
validatorAttempts := 0
|
||||
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
|
||||
if len(lane.extractValidators.validators) == 0 {
|
||||
t.Fatal("extract validators are empty")
|
||||
}
|
||||
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
|
||||
|
||||
var observedProducer, observedValidator *int
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
observedProducer = request.StructuredOutputRepairAttempts
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
observedValidator = target.structuredOutputRepairAttempts
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
|
||||
}
|
||||
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
||||
return err
|
||||
|
||||
@@ -96,7 +96,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
|
||||
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
|
||||
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
|
||||
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
|
||||
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: requestMetadata,
|
||||
LLMProfile: input.pipeline.Chunk.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Chunk.StructuredOutputRepairAttempts), Metadata: requestMetadata,
|
||||
})
|
||||
if callErr != nil {
|
||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
|
||||
|
||||
@@ -420,7 +420,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
|
||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
|
||||
}
|
||||
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
|
||||
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(extractReferences), LLMProfile: lane.Extract.LLMProfile, Metadata: requestMetadata})
|
||||
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(extractReferences), LLMProfile: lane.Extract.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(lane.Extract.StructuredOutputRepairAttempts), Metadata: requestMetadata})
|
||||
if callErr != nil {
|
||||
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
|
||||
return retryAttemptResult{}, terminal.record(nil, attemptErr)
|
||||
|
||||
@@ -41,6 +41,10 @@ type observingChunkValidator struct {
|
||||
request contracts.ChunkValidationRequest
|
||||
}
|
||||
|
||||
type observingSerializedValidator struct {
|
||||
request contracts.SerializedValidationRequest
|
||||
}
|
||||
|
||||
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
||||
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
@@ -50,6 +54,15 @@ func (v *observingChunkValidator) Validate(_ context.Context, request contracts.
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (*observingSerializedValidator) Name() string { return "observing/serialized-validator" }
|
||||
func (*observingSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *observingSerializedValidator) Validate(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
||||
v.request = request
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
||||
|
||||
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
@@ -168,6 +181,33 @@ func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerForwardsDetachedRepairAttemptsToChunkValidators(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||
chunkValidator := &observingChunkValidator{}
|
||||
serializedValidator := &observingSerializedValidator{}
|
||||
chunkAttempts := 2
|
||||
serializedAttempts := 3
|
||||
chunkBinding := Binding(chunkValidator.Name())
|
||||
chunkBinding.StructuredOutputRepairAttempts = &chunkAttempts
|
||||
serializedBinding := Binding(serializedValidator.Name())
|
||||
serializedBinding.StructuredOutputRepairAttempts = &serializedAttempts
|
||||
prepared.chunkValidators.validators = []preparedValidator{
|
||||
{resolved: ResolvedValidator{Binding: chunkBinding, Target: ValidatorTargetChunk}, chunk: chunkValidator},
|
||||
{resolved: ResolvedValidator{Binding: serializedBinding, Target: ValidatorTargetSerialized}, serialized: serializedValidator},
|
||||
}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if got := chunkValidator.request.StructuredOutputRepairAttempts; got == nil || *got != chunkAttempts || got == chunkBinding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("chunk validator repair attempts = %v, want detached value %d", got, chunkAttempts)
|
||||
}
|
||||
if got := serializedValidator.request.StructuredOutputRepairAttempts; got == nil || *got != serializedAttempts || got == serializedBinding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("serialized validator repair attempts = %v, want detached value %d", got, serializedAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
|
||||
|
||||
@@ -258,7 +258,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
if metadataErr != nil {
|
||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("clone merge request metadata: %w", metadataErr))
|
||||
}
|
||||
result, callErr := typed.merge(attemptCtx, typed.merger, contracts.TypedMergeRequest[any]{Source: doc, LaneID: lane.ID, ExtractOutputs: mergeInputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(mergeReferences), LLMProfile: lane.Merge.LLMProfile, Metadata: requestMetadata})
|
||||
result, callErr := typed.merge(attemptCtx, typed.merger, contracts.TypedMergeRequest[any]{Source: doc, LaneID: lane.ID, ExtractOutputs: mergeInputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(mergeReferences), LLMProfile: lane.Merge.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(lane.Merge.StructuredOutputRepairAttempts), Metadata: requestMetadata})
|
||||
if callErr != nil {
|
||||
attemptErr := fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, lane.Merge.Module, callErr)
|
||||
return retryAttemptResult{}, terminal.record(nil, attemptErr)
|
||||
@@ -360,7 +360,7 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
if metadataErr != nil {
|
||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("clone normalize request metadata: %w", metadataErr))
|
||||
}
|
||||
result, callErr := typed.normalize(attemptCtx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(normalizeReferences), LLMProfile: lane.Normalize.LLMProfile, Metadata: requestMetadata})
|
||||
result, callErr := typed.normalize(attemptCtx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(normalizeReferences), LLMProfile: lane.Normalize.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(lane.Normalize.StructuredOutputRepairAttempts), Metadata: requestMetadata})
|
||||
if callErr != nil {
|
||||
attemptErr := fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, lane.Normalize.Module, callErr)
|
||||
return retryAttemptResult{}, terminal.record(nil, attemptErr)
|
||||
@@ -511,6 +511,7 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
|
||||
}
|
||||
requestTarget.value = candidateValue
|
||||
requestTarget.llmProfile = binding.LLMProfile
|
||||
requestTarget.structuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||
result, err = item.typedValidate(validatorCtx, item.typed, requestTarget)
|
||||
case ValidatorTargetSerialized:
|
||||
artifact, encodeErr := validationCandidateArtifact(codec, target)
|
||||
@@ -518,7 +519,7 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
|
||||
err = encodeErr
|
||||
break
|
||||
}
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
|
||||
}
|
||||
|
||||
@@ -28,22 +28,23 @@ type erasedTypedResult struct {
|
||||
}
|
||||
|
||||
type typedValidationTarget struct {
|
||||
stage ModuleStage
|
||||
stepID string
|
||||
laneID string
|
||||
moduleKey string
|
||||
source *source.SourceDocument
|
||||
sourceID string
|
||||
sourceInput contracts.LLMInputMaterial
|
||||
sessionID string
|
||||
references contracts.ReferenceSet
|
||||
llmProfile string
|
||||
metadata map[string]any
|
||||
chunk *source.Chunk
|
||||
chunks []source.Chunk
|
||||
ref source.SourceRef
|
||||
value any
|
||||
candidate *CheckpointArtifact
|
||||
stage ModuleStage
|
||||
stepID string
|
||||
laneID string
|
||||
moduleKey string
|
||||
source *source.SourceDocument
|
||||
sourceID string
|
||||
sourceInput contracts.LLMInputMaterial
|
||||
sessionID string
|
||||
references contracts.ReferenceSet
|
||||
llmProfile string
|
||||
structuredOutputRepairAttempts *int
|
||||
metadata map[string]any
|
||||
chunk *source.Chunk
|
||||
chunks []source.Chunk
|
||||
ref source.SourceRef
|
||||
value any
|
||||
candidate *CheckpointArtifact
|
||||
}
|
||||
|
||||
func exactTypedValue[T any](operation string, value any) (T, error) {
|
||||
|
||||
95
internal/framework/pipeline/typed_registry_request_test.go
Normal file
95
internal/framework/pipeline/typed_registry_request_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type repairObservingNotesMerger struct {
|
||||
attempts *int
|
||||
}
|
||||
|
||||
func (*repairObservingNotesMerger) Key() string { return "test/repair-observing-merge" }
|
||||
|
||||
func (m *repairObservingNotesMerger) Merge(_ context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
||||
m.attempts = request.StructuredOutputRepairAttempts
|
||||
return contracts.TypedMergeResult[codecNotes]{Value: codecNotes{Items: []string{"merged"}}}, nil
|
||||
}
|
||||
|
||||
type repairObservingNotesNormalizer struct {
|
||||
attempts *int
|
||||
}
|
||||
|
||||
func (*repairObservingNotesNormalizer) Key() string { return "test/repair-observing-normalize" }
|
||||
func (*repairObservingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *repairObservingNotesNormalizer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
||||
n.attempts = request.StructuredOutputRepairAttempts
|
||||
return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
func TestTypedRegistryErasurePreservesStructuredOutputRepairAttempts(t *testing.T) {
|
||||
const artifactKind contracts.ArtifactKind = "test/notes"
|
||||
|
||||
t.Run("merge", func(t *testing.T) {
|
||||
implementation := &repairObservingNotesMerger{}
|
||||
registry := NewMergerRegistry()
|
||||
if err := RegisterMerger(registry, ModuleSpec{Key: implementation.Key(), Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Merger[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterMerger() error = %v", err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed merger entry missing")
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
attempts := 2
|
||||
_, err = entry.merge(context.Background(), built, contracts.TypedMergeRequest[any]{
|
||||
ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: codecNotes{Items: []string{"extracted"}}}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("merge() error = %v", err)
|
||||
}
|
||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalize", func(t *testing.T) {
|
||||
implementation := &repairObservingNotesNormalizer{}
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: implementation.Key(), Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed normalizer entry missing")
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
attempts := 3
|
||||
_, err = entry.normalize(context.Background(), built, contracts.TypedNormalizeRequest[any]{
|
||||
MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"merged"}}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -94,6 +94,7 @@ func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
|
||||
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||
binding.Module = strings.TrimSpace(binding.Module)
|
||||
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
|
||||
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||
binding.Options = cloneOptions(binding.Options)
|
||||
if len(binding.References) > 0 {
|
||||
references := make(map[string]ReferenceSource, len(binding.References))
|
||||
@@ -106,6 +107,14 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||
return binding
|
||||
}
|
||||
|
||||
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
|
||||
if attempts == nil {
|
||||
return nil
|
||||
}
|
||||
value := *attempts
|
||||
return &value
|
||||
}
|
||||
|
||||
func cloneReferenceSource(source ReferenceSource) ReferenceSource {
|
||||
out := source
|
||||
if source.Artifact != nil {
|
||||
|
||||
@@ -115,7 +115,7 @@ func RegisterTypedValidatorBuilder[T any](registry *ValidatorRegistry, kind cont
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
||||
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, StructuredOutputRepairAttempts: target.structuredOutputRepairAttempts, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
||||
},
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -45,8 +45,8 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
|
||||
if prepared.SelectedProfileID != "semantic-reconciliation-test" {
|
||||
t.Fatalf("selected profile = %q, want explicit test profile", prepared.SelectedProfileID)
|
||||
}
|
||||
if contract := prepared.OutputContract; contract.SchemaPath != "semantic_reconciliation_llm.v1.json" || contract.RepairAttempts != 0 {
|
||||
t.Fatalf("output contract = %#v, want generic schema without repair", contract)
|
||||
if contract := prepared.OutputContract; contract.SchemaPath != "semantic_reconciliation_llm.v1.json" || contract.RepairAttempts != 1 {
|
||||
t.Fatalf("output contract = %#v, want generic schema with one repair", contract)
|
||||
}
|
||||
if len(prepared.Messages) != 5 || prepared.Messages[0].Role != "system" {
|
||||
t.Fatalf("prepared messages = %#v, want five ordered messages beginning with system", prepared.Messages)
|
||||
|
||||
@@ -43,11 +43,12 @@ func DefaultPromptSpec() (PromptSpec, error) {
|
||||
|
||||
// Request contains one typed owner's source-backed reconciliation input.
|
||||
type Request struct {
|
||||
StageName string
|
||||
Source *source.SourceDocument
|
||||
Candidates []Candidate
|
||||
ProfileID string
|
||||
SessionID string
|
||||
StageName string
|
||||
Source *source.SourceDocument
|
||||
Candidates []Candidate
|
||||
ProfileID string
|
||||
StructuredOutputRepairAttempts *int
|
||||
SessionID string
|
||||
}
|
||||
|
||||
// ResultDisposition classifies a provider-neutral reconciliation outcome.
|
||||
@@ -163,12 +164,13 @@ func (engine *Engine) Reconcile(ctx context.Context, request Request) (Result, e
|
||||
}
|
||||
var response ProposalResponse
|
||||
_, err = engine.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: request.StageName,
|
||||
PromptID: engine.prompt.ID,
|
||||
PromptVersion: engine.prompt.Version,
|
||||
ProfileID: request.ProfileID,
|
||||
SessionID: request.SessionID,
|
||||
Inputs: preparation.Materials(),
|
||||
StageName: request.StageName,
|
||||
PromptID: engine.prompt.ID,
|
||||
PromptVersion: engine.prompt.Version,
|
||||
ProfileID: request.ProfileID,
|
||||
SessionID: request.SessionID,
|
||||
StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts,
|
||||
Inputs: preparation.Materials(),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
|
||||
@@ -95,12 +95,13 @@ func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contrac
|
||||
}
|
||||
var response chunkResponse
|
||||
if _, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: shared.PromptInputs(req.SourceInput, req.References),
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: shared.PromptInputs(req.SourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -209,12 +209,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -143,12 +143,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
}
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -157,7 +157,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
inputs[ItemRegistryReferenceSlot] = registry.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
|
||||
@@ -153,7 +153,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
inputs[LocationRegistryReferenceSlot] = grounding.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
|
||||
@@ -160,12 +160,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -118,12 +118,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -120,12 +120,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -183,12 +183,13 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
inputs[spellcatalog.SpellCatalogReferenceSlot] = e.catalogPromptInput.Clone()
|
||||
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
|
||||
@@ -120,7 +120,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
|
||||
@@ -119,7 +119,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
|
||||
86
scripts/check-release-source.sh
Executable file
86
scripts/check-release-source.sh
Executable file
@@ -0,0 +1,86 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
fail() {
|
||||
printf '%s\n' "check-release-source: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
fail "usage: $0 vMAJOR.MINOR.PATCH"
|
||||
fi
|
||||
|
||||
release_version=$1
|
||||
if ! printf '%s\n' "$release_version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
|
||||
fail "version must be a stable semantic version tag"
|
||||
fi
|
||||
|
||||
script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd)
|
||||
repo_root=$(CDPATH= cd "$script_dir/.." && pwd)
|
||||
cd "$repo_root"
|
||||
|
||||
temporary_dir=$(mktemp -d "${TMPDIR:-/tmp}/notarius-release-source.XXXXXX") || fail "create temporary directory"
|
||||
trap 'rm -rf "$temporary_dir"' 0 HUP INT TERM
|
||||
|
||||
if ! grep -E '^module[[:space:]]+gitea\.maximumdirect\.net/eric/notarius([[:space:]]|$)' go.mod >/dev/null; then
|
||||
fail "go.mod does not declare the expected module path"
|
||||
fi
|
||||
|
||||
if git ls-files --error-unmatch go.work >/dev/null 2>&1 || git ls-files --error-unmatch go.work.sum >/dev/null 2>&1; then
|
||||
fail "tracked Go workspace files are not allowed"
|
||||
fi
|
||||
if [ -d vendor ]; then
|
||||
fail "vendor directory is not allowed"
|
||||
fi
|
||||
if grep -E '^[[:space:]]*replace([[:space:]]|$)' go.mod >/dev/null; then
|
||||
fail "go.mod replace directives are not allowed"
|
||||
fi
|
||||
|
||||
GOWORK=off go test -count=1 ./...
|
||||
GOWORK=off go test -race -count=1 ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./...
|
||||
GOWORK=off go mod tidy -diff
|
||||
|
||||
unformatted=$(git ls-files '*.go' | while IFS= read -r go_file; do
|
||||
fmt_output=$(gofmt -l "$go_file")
|
||||
if [ -n "$fmt_output" ]; then
|
||||
printf '%s\n' "$fmt_output"
|
||||
fi
|
||||
done)
|
||||
if [ -n "$unformatted" ]; then
|
||||
printf '%s\n' "$unformatted" >&2
|
||||
fail "tracked Go files are not gofmt formatted"
|
||||
fi
|
||||
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
|
||||
GOWORK=off go run ./cmd/notarius config validate --config examples/dnd-minimal.config.yml --pipeline dnd-session
|
||||
GOWORK=off OPENROUTER_API_KEY=validation-placeholder go run ./cmd/notarius config validate --config examples/dnd-complete.config.yml --pipeline dnd-session
|
||||
|
||||
host_os=$(GOWORK=off go env GOOS)
|
||||
host_arch=$(GOWORK=off go env GOARCH)
|
||||
version_symbol='gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override'
|
||||
|
||||
build_target() {
|
||||
target_os=$1
|
||||
target_arch=$2
|
||||
target_path="$temporary_dir/notarius-$target_os-$target_arch"
|
||||
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" GOWORK=off go build -trimpath \
|
||||
-ldflags "-X $version_symbol=$release_version" \
|
||||
-o "$target_path" ./cmd/notarius
|
||||
if [ "$host_os" = "$target_os" ] && [ "$host_arch" = "$target_arch" ]; then
|
||||
"$target_path" --version > "$temporary_dir/version-output"
|
||||
printf 'notarius %s\n' "$release_version" > "$temporary_dir/expected-version-output"
|
||||
if ! cmp -s "$temporary_dir/expected-version-output" "$temporary_dir/version-output"; then
|
||||
fail "version output is incorrect for $target_os/$target_arch"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
build_target linux amd64
|
||||
build_target linux arm64
|
||||
build_target darwin amd64
|
||||
build_target darwin arm64
|
||||
Reference in New Issue
Block a user