75 Commits

Author SHA1 Message Date
61436d7c18 Prepare the v0.5.0 release 2026-08-27 19:05:13 +00:00
079d5af337 Harden diagnostic handling and warning presentation 2026-08-27 18:41:59 +00:00
1025001f20 Complete diagnostic migration verification 2026-08-27 16:54:05 +00:00
da14924a02 Document diagnostic ownership and classification 2026-08-27 16:52:58 +00:00
2065a8288b Align debug summaries with diagnostics 2026-08-27 16:51:56 +00:00
af0119cc1d Add diagnostic counts to run receipts 2026-08-27 16:50:32 +00:00
54de2b816a Publish grouped warning and diagnostic files 2026-08-27 16:43:09 +00:00
4dbbf68051 Finalize structured diagnostic aggregation 2026-08-27 16:40:17 +00:00
480680b257 Classify remaining D&D producer diagnostics 2026-08-27 16:04:32 +00:00
6a1fd7bdb6 Classify registry normalization diagnostics 2026-08-27 15:57:29 +00:00
ccba2ce3f9 Classify source relatedness as data-quality advisories 2026-08-27 15:49:53 +00:00
1f1967c8d2 Persist diagnostics in reusable pipeline state 2026-08-27 15:40:45 +00:00
5175cb0722 Classify framework process diagnostics 2026-08-27 15:33:13 +00:00
ba569594a1 Transport structured diagnostics through the pipeline 2026-08-27 15:29:10 +00:00
acb04954eb Add diagnostic contract primitives 2026-08-27 15:19:56 +00:00
610dd3d7c3 Plan warning and diagnostic reform 2026-08-27 15:06:43 +00:00
e92bcfa74c Prepare the v0.4.0 release 2026-08-27 13:30:24 +00:00
e6b2ae88d2 Strengthen validation retries and checkpoint safety 2026-08-27 11:50:37 +00:00
b2e83bd6e7 Document feedback-aware validation retries 2026-08-27 01:35:23 +00:00
7f01c3e79e Add durable validation provenance 2026-08-27 01:27:25 +00:00
7b5f4ebd42 Apply validation retries to merge and normalize 2026-08-27 01:13:32 +00:00
4bb4582695 Integrate extraction validation retries 2026-08-27 01:06:17 +00:00
7d3434e5f6 Expose typed validation reports 2026-08-27 00:37:20 +00:00
29872b2e28 Integrate chunk validation retries 2026-08-27 00:35:37 +00:00
b487d93186 Add producer attempt state machine 2026-08-27 00:27:27 +00:00
8dd7a4324d Execute complete validator chains 2026-08-27 00:18:16 +00:00
04ba87e174 Enable correction-aware semantic reconciliation 2026-08-27 00:08:05 +00:00
a26d6ed042 Enable correction-aware downstream D&D extractors 2026-08-26 23:59:05 +00:00
759d32403f Enable correction-aware foundational D&D producers 2026-08-26 23:55:23 +00:00
1a7b20c766 Adapt corrections through PromptKit 2026-08-26 23:50:24 +00:00
85a5b52be7 Declare producer correction capability 2026-08-26 23:42:11 +00:00
9d0faabf61 Add validation policy configuration 2026-08-26 23:34:06 +00:00
1c3da3e869 Add feedback-aware correction contracts 2026-08-26 23:24:20 +00:00
9abd93502f Upgrade PromptKit to v0.9.0 2026-08-26 23:12:16 +00:00
c8a29a5fa2 Plan feedback-aware validation retries 2026-08-26 18:51:36 +00:00
916d9210fd Preserve structured repair settings across pipeline boundaries 2026-08-25 23:46:52 +00:00
3d3f16db4a Enable structural output repair by default 2026-08-25 20:04:56 +00:00
63c397d86a Resolve structured output repair configuration 2026-08-25 19:55:39 +00:00
9a92212632 Add structured output repair configuration 2026-08-25 19:47:42 +00:00
0ef8931697 Forward repair policy from D&D modules 2026-08-25 19:40:44 +00:00
e00cc45c6b Propagate structured repair requests 2026-08-25 19:39:04 +00:00
ab9b743df6 Add structured output repair support 2026-08-25 19:35:27 +00:00
3bd3c7ebf7 Classify PromptKit generation errors safely 2026-08-25 19:34:27 +00:00
75a3f51cee Support inherited PromptKit profiles 2026-08-25 19:31:00 +00:00
2be999ebd3 Verify PromptKit compatibility boundaries 2026-08-25 19:27:34 +00:00
32fe7c5b98 Upgrade PromptKit to v0.8.0 2026-08-25 19:25:57 +00:00
55247c47ab Plan the PromptKit 0.8 upgrade 2026-08-25 19:11:30 +00:00
5e5c69bf9d Document future validation and retry work 2026-08-25 14:29:00 +00:00
b4a81f8b09 Validate release versions before tagging 2026-08-25 02:21:46 +00:00
adfed22e1a Complete source release system verification 2026-08-25 02:05:17 +00:00
75b1e2f68b Document source release deployment 2026-08-25 02:03:11 +00:00
4473363d9f Document source release procedure 2026-08-25 02:01:52 +00:00
6eb45e0003 Add release tag validation workflow 2026-08-25 01:58:29 +00:00
0585ad76dc Add source release candidate checks 2026-08-25 01:57:10 +00:00
56145c3b7e Add build version reporting 2026-08-25 01:55:09 +00:00
a478fd86c5 Plan source-only releases 2026-08-25 01:48:16 +00:00
916532100d Add D&D consumer documentation 2026-08-09 20:57:28 +00:00
bef8ca263b Tighten evidence context source excerpts 2026-08-09 20:04:46 +00:00
f6d037b613 Document evidence context source unit excerpts 2026-08-09 19:47:27 +00:00
449b506804 Update evidence context output coverage 2026-08-09 19:45:21 +00:00
071a78ae22 Replace evidence context with source unit excerpts 2026-08-09 19:42:33 +00:00
ad1cba41c2 Fix OpenAI reconciliation schema 2026-08-09 18:49:29 +00:00
67338798aa Complete the semantic reconciliation roadmap 2026-08-09 18:30:32 +00:00
e95e2f2220 Finalize semantic reconciliation documentation 2026-08-09 17:05:17 +00:00
628b8d1800 Remove legacy reconciliation path 2026-08-09 16:57:44 +00:00
d24d4609b6 Migrate location reconciliation to shared engine 2026-08-09 16:51:29 +00:00
c7f79fb38e Migrate item reconciliation to shared engine 2026-08-09 16:46:43 +00:00
8c071800cf Migrate NPC reconciliation to shared engine 2026-08-09 16:38:43 +00:00
569e12c6f4 Add generic reconciliation plan application 2026-08-09 16:27:36 +00:00
5b6eb591b2 Add shared semantic reconciliation engine 2026-08-09 16:19:54 +00:00
b630384aa0 Add generic semantic reconciliation prompt assets 2026-08-09 16:08:53 +00:00
297d58f090 Add semantic reconciliation proposal validation 2026-08-09 16:00:01 +00:00
ee71dc4937 Add bounded semantic candidate preparation 2026-08-09 15:52:55 +00:00
65e5d65d14 Record semantic reconciliation architecture decision 2026-08-09 15:43:05 +00:00
b40b40aaf3 Plan semantic reconciliation improvements 2026-08-09 15:38:10 +00:00
338 changed files with 16812 additions and 4902 deletions

33
.woodpecker/release.yml Normal file
View File

@@ -0,0 +1,33 @@
when:
- event: tag
steps:
- name: validate-release
image: golang:1.25.5
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
release_note="docs/releases/$version.md"
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
printf '%s\n' "invalid release tag: $version" >&2
exit 1
fi
if [ ! -s "$release_note" ]; then
printf '%s\n' "missing release note: $release_note" >&2
exit 1
fi
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
printf '%s\n' "release note heading does not match $version" >&2
exit 1
fi
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
printf '%s\n' "release note is missing heading: $heading" >&2
exit 1
fi
done
./scripts/check-release-source.sh "$version"

View File

@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
[its synthetic transcript](examples/dnd-complete-transcript.json). It
demonstrates all implemented D&D lanes and the supporting campaign references.
## Install A Source Release
Install a pinned source release with Go:
~~~
GOWORK=off go install \
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
~~~
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
installed command's diagnostic version is described in the [CLI
reference](docs/cli.md); maintainers preparing a release should follow [Source
Releases](docs/release.md).
## Documentation
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
artifact formats.
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
from an orchestrator and consume a published result.
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
D&D pipeline as a subprocess and discover its structured artifacts.
- [Internal overview](docs/internal/overview.md) — implemented component map
for maintainers.
- [Developer guide](docs/development.md) — contributor orientation and

View File

@@ -42,4 +42,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_combat_turns_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -50,4 +50,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_enemy_events_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,47 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.dnd.entity_reconcile.llm",
"type": "object",
"additionalProperties": false,
"required": ["duplicate_groups"],
"properties": {
"duplicate_groups": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["members", "canonical"],
"properties": {
"members": {
"type": "array",
"items": {"$ref": "#/$defs/selector"}
},
"canonical": {"$ref": "#/$defs/selector"}
}
}
}
},
"$defs": {
"selector": {
"type": "object",
"additionalProperties": false,
"required": ["name", "source_refs"],
"properties": {
"name": {"type": "string", "minLength": 1},
"source_refs": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"additionalProperties": false,
"required": ["start_unit_id", "end_unit_id"],
"properties": {
"start_unit_id": {"type": "integer", "minimum": 1},
"end_unit_id": {"type": "integer", "minimum": 1}
}
}
}
}
}
}
}

View File

@@ -42,4 +42,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_item_occurrences_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_item_registry_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,8 +1,9 @@
Use candidate names and cited transcript windows only to determine whether
candidates identify the same item type or unique designation. Do not treat
nearby evidence, similar objects, or a shared owner as sufficient. Keep
currency denominations, materially different item types, and uncertain aliases
separate. Do not infer an item property or uniqueness.
Determine whether candidates identify the same item type or unique designation
using their contextual labels and cited transcript windows. Do not treat nearby
evidence, similar objects, or a shared owner as sufficient.
Keep currency denominations and materially different item types separate. Keep
uncertain aliases separate. Do not infer an item property or uniqueness.
When selecting a canonical display name, choose one supplied candidate name
that is the clearest established designation.

View File

@@ -12,19 +12,19 @@ messages:
- role: system
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./instructions.md
content_file: ./sharedassets/protocol.md
- role: user
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
content_file: ./instructions.md
cache_control:
type: ephemeral
- role: user
content_file: ./candidates.md
content_file: ./sharedassets/candidates.md
- role: user
content_file: ./sharedassets/common-dnd-transcript-windows.md
content_file: ./sharedassets/transcript-windows.md
cache_control:
type: ephemeral
output:
format: json
validation_mode: json_schema
schema_path: dnd_entity_reconcile_llm.v1.json
repair_attempts: 0
schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 1

View File

@@ -42,4 +42,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_location_occurrences_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_location_registry_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,2 +0,0 @@
Location candidates:
{{ input "candidates" }}

View File

@@ -1,6 +1,8 @@
Use candidate names and their cited transcript windows to determine whether
candidates identify the same physical place. Do not treat matching names,
nearby evidence, nested places, or generic labels as sufficient. Keep parent
and child places, similarly named places, and uncertain aliases separate.
Determine whether candidates identify the same physical place using their
contextual labels and cited transcript windows. Do not treat matching names,
nearby evidence, nested places, or generic labels as sufficient.
Keep parent and child places separate, as well as similarly named places and
uncertain aliases.
When selecting a canonical display name, prefer the clearest established name.

View File

@@ -12,19 +12,19 @@ messages:
- role: system
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./instructions.md
content_file: ./sharedassets/protocol.md
- role: user
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
content_file: ./instructions.md
cache_control:
type: ephemeral
- role: user
content_file: ./candidates.md
content_file: ./sharedassets/candidates.md
- role: user
content_file: ./sharedassets/common-dnd-transcript-windows.md
content_file: ./sharedassets/transcript-windows.md
cache_control:
type: ephemeral
output:
format: json
validation_mode: json_schema
schema_path: dnd_entity_reconcile_llm.v1.json
repair_attempts: 0
schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 1

View File

@@ -42,4 +42,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_npc_occurrences_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -37,4 +37,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_npc_registry_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,3 +0,0 @@
NPC candidates for identity comparison:
{{ input "candidates" }}

View File

@@ -1,6 +1,7 @@
Use candidate aliases and their cited transcript windows to determine whether
candidates refer to the same individual. Preserve distinct individuals even
when their names are similar.
Determine whether candidates refer to the same individual using their
contextual labels and cited transcript windows. Preserve distinct individuals
even when their names are similar or their contextual descriptions are
identical.
When selecting a canonical display name, prefer a complete, stable proper name
over an abbreviation. Prefer an unadorned proper name over that name plus a

View File

@@ -12,19 +12,19 @@ messages:
- role: system
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./instructions.md
content_file: ./sharedassets/protocol.md
- role: user
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
content_file: ./instructions.md
cache_control:
type: ephemeral
- role: user
content_file: ./candidates.md
content_file: ./sharedassets/candidates.md
- role: user
content_file: ./sharedassets/common-dnd-transcript-windows.md
content_file: ./sharedassets/transcript-windows.md
cache_control:
type: ephemeral
output:
format: json
validation_mode: json_schema
schema_path: dnd_entity_reconcile_llm.v1.json
repair_attempts: 0
schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 1

View File

@@ -35,4 +35,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_scene_descriptions_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -31,4 +31,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_scenes_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,7 +0,0 @@
Identify only well-supported duplicate groups among the supplied candidates.
Return each selected candidate's supplied contextual descriptor exactly: its
`name` and complete ordered `source_refs`. A group must contain at least two
supplied descriptors, and its `canonical` descriptor must be one of its
members. Do not invent names, ranges, records, evidence, or replacement values.
Omit any uncertain or unsafe group.

View File

@@ -1,6 +0,0 @@
Selected Dungeons & Dragons gameplay transcript evidence windows are provided
below. They may be incomplete, non-contiguous, or overlapping. Use them to
evaluate candidate identity, but do not treat absence outside these windows as
evidence.
{{ input "transcript" }}

View File

@@ -47,4 +47,4 @@ output:
format: json
validation_mode: json_schema
schema_path: dnd_spells_llm.v1.json
repair_attempts: 0
repair_attempts: 1

View File

@@ -1,3 +0,0 @@
Candidates for identity comparison:
{{ input "candidates" }}

View File

@@ -1,5 +0,0 @@
Your task is to identify duplicate entities withi the provided list of candidates.
Please review the listed candidates and their underlying evidence, and determine whether any of the listed candidates refer to the same underlying entity. Preserve distinct candidates that appear to refer to different underlying entities, even when their names are similar.
When selecting a canonical display name, prefer a complete, stable proper name over an abbreviation. Prefer an unadorned proper name over that name plus additional descriptors, unless the underlying evidence stablishes the descriptors as part of the entity's name. A longer display name is not inherently more canonical.

View File

@@ -1,2 +1,3 @@
Item candidates:
Candidate material:
{{ input "candidates" }}

View File

@@ -0,0 +1,5 @@
Identify only high-confidence duplicate entities among the supplied candidates.
Preserve distinct entities even when their names are similar. Treat contextual descriptions and transcript evidence as supporting material, not as permission to merge ambiguous records.
When several records are duplicates, choose as canonical the candidate with the clearest stable identity. Prefer a complete proper name over an abbreviation, and prefer an unadorned proper name over one with incidental descriptors unless the evidence establishes those descriptors as part of the name. A longer name is not inherently more canonical.

View File

@@ -0,0 +1,27 @@
id: generic.semantic_reconciliation
version: "v1"
inputs:
- name: candidates
required: true
content_type: application/json
- name: transcript
required: true
content_type: application/json
messages:
- role: system
content_file: ./system.md
- role: user
content_file: ./protocol.md
- role: user
content_file: ./instructions.md
cache_control:
type: ephemeral
- role: user
content_file: ./candidates.md
- role: user
content_file: ./transcript-windows.md
output:
format: json
validation_mode: json_schema
schema_path: semantic_reconciliation_llm.v1.json
repair_attempts: 1

View File

@@ -0,0 +1,7 @@
Use only the positive integer `candidate_id` values supplied in the candidate material.
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group.
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid.
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.

View File

@@ -0,0 +1,2 @@
You reconcile structured records that may describe the same underlying entity.
Follow the supplied protocol and return only the requested structured result.

View File

@@ -0,0 +1,3 @@
Transcript evidence windows:
{{ input "transcript" }}

View File

@@ -0,0 +1,32 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.generic.semantic_reconciliation.llm",
"title": "notarius_semantic_reconciliation_llm_v1",
"type": "object",
"additionalProperties": false,
"required": ["duplicate_groups"],
"properties": {
"duplicate_groups": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["candidate_ids", "canonical_candidate_id"],
"properties": {
"candidate_ids": {
"type": "array",
"minItems": 2,
"items": {
"type": "integer",
"minimum": 1
}
},
"canonical_candidate_id": {
"type": "integer",
"minimum": 1
}
}
}
}
}
}

View File

@@ -0,0 +1,89 @@
# ADR-0013: Use request-local candidate handles for semantic reconciliation
**Status:** Accepted
**Date:** 2026-08-09
## Context
Several typed normalize stage modules need semantic reconciliation after
deterministic preprocessing: a model can judge whether source-backed candidates
refer to the same underlying entity, while application code remains responsible
for constructing the normalized artifact. Requiring the model to reproduce a
candidate's full contextual selector makes the response larger and introduces
avoidable formatting, ordering, and transcription failure modes.
Reconciliation must preserve the exact typed artifact boundary established by
[ADR-0003](0003-typed-interfaces-with-two-zone-data-model.md), the domain-neutral
framework and concrete-domain dependency direction established by
[ADR-0004](0004-package-modules-by-domain.md), and the distinction in
[ADR-0009](0009-minimal-evidence-grounded-extraction-artifacts.md) between source
evidence and auxiliary identity context. It also needs a concrete, narrowly
scoped application of the request-local-label exception allowed by
[ADR-0012](0012-resolve-opaque-entity-identifiers-deterministically.md).
## Decision
Semantic reconciliation will be a domain-neutral framework mechanism used by
typed normalize stage modules. A consuming artifact family will retain
ownership of its typed records, identity rules, consolidation policy, durable
IDs, and domain warnings; the framework mechanism will not infer those rules
from arbitrary data.
For each reconciliation request, deterministic code will assign every eligible
model-visible candidate a contiguous, one-based integer handle. The model may
receive the candidate's contextual label, source references, and bounded source
context needed to judge identity, but its structured response will identify
candidates only by those supplied handles. A handle is local to one request,
does not represent entity identity, and must never enter a durable artifact or
be used to derive a durable ID.
The model will propose duplicate groups and select one supplied member of each
group as canonical. Deterministic code will resolve the handles through the
retained request mapping, validate the complete proposal, discard unsafe
groups, and apply only validated groups through typed domain-owned policy. The
model will not synthesize replacement records or directly mutate an artifact.
Every reconciliation prompt will combine a mandatory framework-owned protocol
and safety policy with an explicitly selected semantic policy. The semantic
policy may be the conservative generic policy or a domain-owned policy, but it
cannot replace the shared response protocol or deterministic safety boundary.
## Alternatives considered
- Return durable application IDs. Opaque IDs do not help semantic judgment,
expose application identity mechanics, and make model output reproduce data
that deterministic code already owns.
- Return names alone or copied contextual selectors. Names can be ambiguous,
while reproducing labels and source ranges adds response complexity and
creates mismatches without adding semantic information. Request-local
handles preserve exact selection without either failure mode.
- Ask the model to return synthesized canonical replacement records. This
would transfer typed artifact construction, provenance consolidation, and
durable identity policy to a probabilistic boundary.
- Reconcile reflection-discovered fields or arbitrary JSON. This would weaken
the typed artifact contract and move domain semantics into generic code.
- Hide reconciliation inside extraction or another stage. This would obscure
stage ownership and create cross-stage behavior outside the fixed pipeline;
reconciliation remains explicit normalize-stage behavior.
- Let each domain replace the complete prompt protocol. This would duplicate
safety mechanics and allow domain policy to bypass the common response and
validation contract.
## Consequences
Model responses become smaller and easier to validate, while deterministic
application code retains authority over identity, provenance, ordering, and
typed artifact construction. The framework requires a request-local mapping,
bounded context preparation, a private integer response contract, proposal
assessment, and shared prompt assets. Each consuming artifact family still
requires a typed adapter for its irreducibly domain-specific rules.
Request-local handles are deliberately unsuitable for persistence, logging as
entity identity, checkpoint contracts, or cross-request correlation. Changes
to shared protocol and policy assets must participate in the normal prompt,
schema, and checkpoint fingerprint mechanisms.
The shared mechanism and its initial D&D registry consumers are now
implemented. Current behavior is documented in
[Module Internals](../internal/modules.md#semantic-reconciliation) and
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).

View File

@@ -0,0 +1,59 @@
# ADR-0014: Use feedback-aware validation retries
**Status:** Accepted
**Date:** 2026-08-26
## Context
Validation can identify a candidate defect after a producer has returned an
otherwise well-formed result. Retrying without the validator's deterministic,
bounded feedback wastes the useful diagnosis, while treating validator
execution failures as defects would ask a producer to repair conditions it
cannot control. The mechanism must preserve typed producer ownership,
checkpoint safety, and the repository's sensitive-data boundaries.
## Decision
The implementation will keep three independent budgets: the producer binding's
outer `retries` budget, PromptKit's structured-output repair budget, and each
validator's execution-retry budget. Validators will run sequentially in their
configured order and aggregate both rejections and execution failures before a
candidate disposition is selected.
A correction-capable producer will provide the exact single LLM response that
controlled its candidate using the `single_response_v1` protocol. A correction
attempt will reconstruct the ordinary request and append exactly two fresh
messages: that latest response as `assistant`, followed by one deterministic
aggregate correction request as `user`. Earlier turns will not accumulate.
Validator failures will not recurse into correction. Pipeline policy owns
terminal disposition, with field-by-field producer overrides over pipeline
defaults: structural failure and semantic rejection default to `fail_run`, and
validator execution failure defaults to `warn_continue`. Validators can report
facts and bounded corrective guidance, but never decide disposition.
Rejected and structurally invalid candidates will not advance. A candidate
allowed through after a validator execution failure will retain explicit
incomplete-validation provenance and will not be checkpointed. Exact response
and correction text remain attempt-local: they are excluded from ordinary
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
summaries.
## Alternatives considered
- Retry every producer after any validation outcome. This conflates producer
defects with validator operational failures and wastes retry budget.
- Let validators decide whether to continue. This would distribute pipeline
disposition policy across validators and undermine consistent defaults.
- Reuse the full prior conversation. Accumulated turns introduce unbounded
prompt growth and make correction behavior depend on incidental history.
- Persist raw responses to simplify diagnosis. Raw model output and correction
guidance may be sensitive and do not belong in durable pipeline records.
## Consequences
The framework gains transport-neutral correction and candidate contracts,
producer capability checks, policy resolution, aggregated validation outcomes,
and conservative checkpoint handling. Prompt construction remains inside the
LLM adapter, while modules remain responsible for accurately exposing the
single response that directly controlled a candidate.

View File

@@ -0,0 +1,67 @@
# ADR-0015: Separate process warnings from quality diagnostics
**Status:** Accepted
**Date:** 2026-08-27
## Context
Notarius currently represents process degradation, incomplete validation,
extraction-quality doubt, and routine normalization with one flat warning
record. That makes ordinary successful runs noisy, loses the framework context
needed to explain a finding, and gives `warning_count` no stable operational
meaning. It also permits output encoders to add a warning after the durable
warning file has already been written.
The application needs one bounded diagnostic model that preserves exact
occurrence counts while retaining only safe, representative samples. Fresh and
resumed logical runs must present the same groups. The model must not alter
validation decisions, retry budgets, rejected-output behavior, or process exit
policy.
## Decision
Warnings are reserved for a completed run that advanced under an allowed
process-level degradation or incomplete-work policy. Extraction-quality signals
are advisories, and routine accepted transformations are observations. A
non-degraded successful run therefore has zero actionable warnings.
Modules and validators own a diagnostic's disposition, category, reason code,
scope, and safe message. The framework adds pipeline origin, including stage,
step, lane, module, validator, and chunk context where applicable. It then
aggregates deterministically by disposition, category, reason code, and full
origin. Chunk context remains on representative samples so equivalent findings
across chunks aggregate together.
Diagnostics carry exact occurrence counts, at most three distinct samples, and
numeric omitted-sample metadata. Producers and validators are bounded to 64
local groups. Final actionable warning groups are bounded without truncation;
the non-warning collection may truncate represented groups while preserving an
exact total occurrence count and explicit truncation metadata.
The public contracts will be versioned: grouped actionable warnings use
`notarius.warnings.v2`, grouped advisories and observations use
`notarius.diagnostics.v1`, and the run receipt uses
`notarius.run-result.v2`. Successful output encoders return logical files or
an error; they do not add post-encoding warnings.
## Alternatives considered
- Keep one warning list and filter only CLI output. This would leave durable
consumers with the same semantically mixed, unbounded contract.
- Map reason codes to severity in a central framework registry. This would
split module-owned meaning between synchronized policy tables and make new
diagnostic meaning implicit.
- Preserve local omission warning records. They inflate visible group counts
and lose exact occurrence semantics.
- Keep output-encoder warnings. A one-pass encoder cannot include those
records consistently in files it has already serialized; a two-phase encoder
protocol is deferred until a demonstrated need exists.
## Consequences
The framework gains validated diagnostic primitives, local collection,
origin-aware aggregation, and versioned durable presentation. Existing warning
transport remains temporarily while producers migrate. Current architecture,
operator, integration, and internal documentation will describe the behavior
only as each implementation step lands; this accepted decision does not claim
that the migration is complete.

View File

@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
~~~
notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--json]
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
writes the command summary to standard output and exits with status 0.
`notarius --version` is valid only as the sole root argument. It writes exactly
`notarius <version>` followed by a newline to standard output and exits with
status 0. A tagged `go install` build can report its main-module stable tag,
and controlled builds can inject a stable tag at link time through
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
unversioned checkout reports `development`. Invalid injected version content is
a runtime error with exit status 1, while extra `--version` arguments are a
syntax error with exit status 2. This diagnostic does not replace the
[run-result](integrations/run-result.md) or artifact contracts for downstream
compatibility decisions.
## run
~~~
@@ -91,11 +103,14 @@ names, requiredness, and configured bindings are part of the
Without **--json**, standard output contains the completed pipeline ID, counts
of normalized and rejected outputs, and the output directory. A debug-enabled
run also prints its debug-bundle path to standard output. A successful run with
warnings reports the warning count to standard error. The published JSON bundle
actionable process warnings reports their group and occurrence counts to
standard error. When the selected output module publishes `warnings.json`, the
summary also reports that durable file's path. Advisory and observation findings
do not produce a warning line. The published JSON bundle
is defined by the [JSON output contract](integrations/json-output.md).
With **--json**, successful standard output is exactly one
`notarius.run-result.v1` JSON document followed by a newline, with no
`notarius.run-result.v2` JSON document followed by a newline, with no
human-oriented status or debug-path line. Its fields and compatibility policy
are defined by the [run-result contract](integrations/run-result.md). A caller
must check for exit status 0 before decoding this output; a failed write can
@@ -159,7 +174,7 @@ go run ./cmd/notarius pipelines list \
Successful commands write their primary result to standard output. Warnings and
errors are written to standard error.
For **run --json**, warnings remain on standard error and standard output is a
For **run --json**, actionable process warnings remain on standard error and standard output is a
machine-readable success result only. Syntax and runtime diagnostics remain on
standard error. Parse the result only after the process exits with status 0.

View File

@@ -132,7 +132,11 @@ model: example-model
Keep credentials out of the local-backend object. A PromptKit profile may name
its credential environment variable through `api_key_env`; set that variable
only in the run environment. PromptKit owns the
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
including `base_profile` inheritance. Notarius passes profiles through without
merging them. Filesystem profiles cannot express PromptKit's in-memory
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
provider without authorization.
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
supported package API, and [Operations](operations.md#operational-limits)
describes the effective concurrency layers.
@@ -218,6 +222,8 @@ pipelines:
| Field | Type | Default | Rules |
| --- | --- | --- | --- |
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
| **input** | module binding | none | Required. |
| **chunk** | module binding | **generic** | Optional. |
| **output** | module binding | **json** | Optional. |
@@ -237,6 +243,42 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
then the pipeline's **llm_profile**, and finally the PromptKit default.
Deterministic bindings do not receive these defaults or run overrides.
Structural output repair is resolved after module, validator, and `--only` lane
selection. An object's **structured_output_repair_attempts** value takes
precedence over the pipeline value; otherwise, an LLM-backed binding or
validator inherits the pipeline value. If both are omitted, PromptKit uses the
prompt's declared repair policy. The value must be an integer from 0 through 3;
explicit `null` and non-integer values are invalid. An explicit value on a
deterministic binding or validator is invalid, while a pipeline value simply
does not apply to deterministic selections.
`validation_policy` controls terminal disposition for one complete producer
attempt and validator chain. It may appear on a pipeline or a **chunk**,
**extract**, **merge**, or **normalize** module binding; input, output, and
validator bindings reject it. Every field is optional and resolves in binding,
pipeline, then application-default order:
| Field | Values | Default |
| --- | --- | --- |
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
The policy object and its fields must be non-null, and unknown fields are
rejected. A deterministic producer may not explicitly set
**producer_structural_failure** on its binding, although a pipeline-level
default remains valid for pipelines that include LLM-backed producers.
After the producer binding's retry budget is exhausted, an invalid structured
response uses **producer_structural_failure**. One or more semantic validator
rejections use **semantic_rejection**; rejection takes precedence over an
exhausted validator failure or skip. With no rejection, an exhausted validator
failure or skip uses **validator_failure**. `reject_output` records the
terminal rejection without advancing that candidate. `warn_continue` is valid
only for validator execution failure: it advances a structurally valid,
otherwise unrejected result with incomplete-validation provenance and without
making it reusable checkpoint state.
A lane has these fields:
| Field | Type | Default | Rules |
@@ -274,17 +316,32 @@ extract:
| --- | --- | --- | --- |
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
| **options** | object | none | Must satisfy the selected module. |
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
Omitting **validators** uses the registered chain. **validators: []** selects
an empty chain; a non-empty list replaces the chain in the listed order.
Validator bindings accept only **module**, **llm_profile**, and **options**.
They reject **references**, **retries**, and nested **validators**. Deterministic
validators reject an explicit **llm_profile**. Deterministic module bindings
also reject an explicit **llm_profile**.
Validator bindings accept only **module**, **llm_profile**,
**structured_output_repair_attempts**, **retries**, and **options**. Their
**retries** value is a non-negative additional validator-execution budget and
is valid only when the selected validator is LLM-backed. A validator retry
rechecks the same immutable candidate; it never regenerates the producer.
They reject
**validation_policy**, **references**, and nested **validators**. Deterministic
validators reject explicit **llm_profile** and
**structured_output_repair_attempts**.
Deterministic module bindings also reject those explicit fields.
An LLM-backed chunk, extract, merge, or normalize producer with both a
non-empty validator chain and positive **retries** must declare the supported
single-response correction capability. Preparation rejects a configuration
that could require semantic correction from a producer that cannot provide an
exact prior response. A deterministic producer, or an LLM attempt that did
not make a model call, cannot consume a semantic retry after rejection.
The **json** output module accepts optional **include_chunk_map** and
**evidence_context** settings:
@@ -319,7 +376,8 @@ Unknown outer or nested option fields are rejected, as are incompatible YAML
types. The allowlist remains valid when a run uses lane filtering: a configured
lane that is not active for that invocation simply contributes no evidence.
Evidence publication is opt-in because it can persist source text and metadata.
Its payload contract is [Published Evidence Context](integrations/evidence-context.md).
When enabled, it publishes the selected source-unit excerpt defined by the
[Published Evidence Context contract](integrations/evidence-context.md).
## References And Ordered Handoffs

View File

@@ -0,0 +1,209 @@
# Consuming The Complete D&D Pipeline
Use this workflow when an orchestrator runs the maintained complete D&D
pipeline and consumes its structured JSON artifacts. The generic
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
this guide connects that workflow to the complete D&D configuration, its
Seriatim input, and its artifact inventory.
The [CLI reference](../cli.md), [configuration reference](../config.md),
[run-result receipt](../integrations/run-result.md), and
[published JSON output contract](../integrations/json-output.md) remain the
canonical definitions of those public interfaces.
## Prepare And Validate The Deployment
Start from the maintained
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
artifact handoffs, campaign references, chunk-map publication, and evidence
context.
A deployment must provide its own PromptKit profile and campaign reference
files. Use absolute paths for service and subprocess deployments. In
particular, observe these different resolution rules:
- reference paths in YAML are resolved relative to the Notarius configuration
file; and
- `promptkit.profile_file` is resolved relative to the Notarius process working
directory.
Do not copy the repository example's relative profile path into a deployment
without also controlling that working directory. The complete path and profile
rules are defined in [Configuration](../config.md).
Preflight the deployed configuration before processing sessions and whenever
it changes:
```sh
notarius config validate \
--config /absolute/path/to/notarius.yml \
--pipeline dnd-session
```
Provide credentials through the environment or the documented configuration
mechanism. Do not put credentials in command arguments, generated
configuration, or logs.
## Supply The Transcript
The complete pipeline consumes a Seriatim JSON document. The
[Seriatim input contract](../integrations/seriatim.md) defines its required
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
citations use those segment IDs as source-unit ranges.
When the caller maintains several transcript tiers, use the final trimmed JSON
transcript so extraction operates on the same session content presented to
later consumers. For example, Narratio identifies this implemented artifact as
`narratio.transcript.final_trimmed` and normally stores it at
`transcripts/final.trimmed.json`.
Notarius generates a stable prompt session from the resolved input module and
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
Use that override only when intentionally changing the routing relationship
between invocations; it is not a credential or output identity.
## Run Notarius
Invoke the pipeline with explicit absolute paths and request its
machine-readable receipt:
```sh
notarius run dnd-session \
--config /absolute/path/to/notarius.yml \
--input /absolute/path/to/transcripts/final.trimmed.json \
--output-dir /absolute/path/to/notarius-output \
--json
```
The caller should:
- capture stdout and stderr separately;
- propagate cancellation and impose an operator-appropriate timeout;
- wait for process completion before interpreting stdout; and
- retain stderr for diagnosis without copying secrets or transcript content
into other logs.
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
nonzero exit because a failed receipt write can leave partial bytes. The
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
stream and exit-status contract.
## Discover The Published Bundle
Decode the successful stdout document as a supported run-result schema. For
the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate
unknown fields allowed by that version, but reject an unsupported schema
version.
Use the receipt's absolute `output_directory` as the exact run-specific bundle
root. Do not scan the output root for its newest directory, guess a run ID, or
construct a bundle path. Resolve `index_file` beneath `output_directory` and
reject an absolute logical path or any result that escapes the bundle root.
The complete configuration uses the application validation defaults. A caller
that requires fully validated D&D artifacts must also require receipt
`validation_status: approved`; a successful `incomplete` result reflects the
configured validator-failure continuation policy and carries its bounded
validator provenance in `validation_summaries`.
Read `index.json` and locate each requested lane in `output_files` by its exact
`lane_id`. Do not guess a lane filename. Before decoding a payload:
1. resolve its descriptor's relative `file` beneath the bundle root with the
same confinement check;
2. verify the descriptor's media type and schema identity against the linked
artifact contract; and
3. decode the payload according to that contract.
The [published JSON output contract](../integrations/json-output.md) defines
the index and bundle layout. Treat all paths obtained from a decoded external
document as untrusted until confined to their documented root.
## Complete Artifact Inventory
When every configured lane is accepted, the complete example publishes these
lane artifacts:
| Lane ID | Purpose | Canonical contract |
| --- | --- | --- |
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
The JSON encoder always publishes these bundle-management files:
| File | Purpose |
| --- | --- |
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
| `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Advisory and observation findings for accepted artifacts. |
The complete configuration also requests two pipeline-wide artifacts:
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
chunk metadata; and
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
excerpt containing the union of selected cited source units and the
configured surrounding window.
Discover both from their top-level `index.json` descriptors rather than
treating them as lanes. Evidence context is convenient reading material, not
authoritative provenance; citations in the normalized lane payloads remain the
evidence contract.
Every optional or lane file is published only when its corresponding artifact
is available. A successful process does not guarantee that all configured
lanes were accepted.
## Decide What Counts As Consumer Success
Exit status 0 means Notarius completed the pipeline and published its result
bundle. The receipt or bundle may still report warnings, rejected outputs, or
missing lane descriptors. A downstream consumer must define its own required
artifact set explicitly.
A caller that claims to consume the complete D&D workflow should normally
require all ten lane IDs in the table and verify each descriptor's expected
contract. If any required lane is missing, rejected, or incompatible, fail the
caller's extraction step while retaining the Notarius bundle for diagnosis. A
consumer that needs only a subset may define and document a narrower policy.
Keep the successful receipt with the complete published bundle. Retain
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
required by the caller's provenance, diagnosis, and retention policies. Avoid
selectively copying payload files without also preserving enough index and
manifest information to identify their originating run and contracts.
The transcript, lane artifacts, evidence context, manifest, debug data, and
logs can all contain private campaign information. Apply the same access,
publication, and retention controls used for the source transcript.
## Consumer Checklist
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
- Use absolute configuration, input, output-root, profile, and reference paths
in service deployments.
- Capture stdout and stderr separately and enforce cancellation and timeout.
- Parse stdout only after exit status 0.
- Accept only supported receipt, index, and artifact schema versions while
tolerating permitted unknown fields.
- Use the receipt's `output_directory`; never guess the run directory.
- Confine `index_file` and every descriptor path to the published bundle root.
- Discover lanes by `lane_id` and verify descriptor compatibility before
decoding payloads.
- Enforce an explicit required-lane policy and inspect rejections and warnings.
- Preserve the receipt and sufficient bundle provenance for every retained
artifact.
- Protect all transcript-derived files and diagnostic streams as sensitive
campaign data.

View File

@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
[Published JSON Output contract](../integrations/json-output.md) own the
durable result formats.
For the maintained complete D&D workflow, including its transcript input,
configured lane inventory, and downstream acceptance checklist, see
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
## Run And Check The Process
Optionally preflight a selected configuration and pipeline before work starts:
@@ -55,23 +59,30 @@ contract. The JSON bundle contract links to the available lane contracts.
If `index.json` has an `evidence_context` descriptor, treat it as a
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
fields before decoding the linked file according to the [Published Evidence
Context contract](../integrations/evidence-context.md). Use each
`evidence_refs` entry as the citation to source material. Its surrounding
context range and included units explain the citation, but do not widen or
replace the cited source reference.
Context contract](../integrations/evidence-context.md). Decode its top-level
source-unit array as a reading excerpt. Obtain authoritative citations and lane
provenance from the normalized lane artifacts; the excerpt has neither and its
nearby units do not widen a lane artifact's cited source reference.
A zero exit status may still report rejected outputs, warnings, or absent
lanes. The caller decides which lane IDs are required for its own work and
which are optional; it should make that decision explicitly rather than infer
failure from the receipt counts alone.
When complete validation is required, also require receipt
`validation_status: approved` and inspect `validation_summaries`. A successful
run with `validation_status: incomplete` contains a structurally valid result
that advanced after validator execution could not complete under the configured
`warn_continue` policy. It is not reusable checkpoint state and should not be
silently treated as fully reviewed by the caller.
## Preserve Provenance And Handle Data Carefully
Keep the receipt with the published `manifest.json`, and retain
`rejected.json` and `warnings.json` when review or later provenance requires
`rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires
them. Treat the input, output bundle, cache, debug bundle, and captured process
logs as potentially sensitive data. Apply the caller's access controls and
retention policy, and avoid copying secrets into arguments, logs, or
provenance records. An evidence-context artifact contains source-unit text and
metadata, and selected lanes can cover most of an input; preserve and share it
only when that source content is authorized for the recipient.
metadata and can cover most of an input; preserve and share it only when that
source content is authorized for the recipient.

View File

@@ -18,13 +18,14 @@ implemented component map.
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and the package-local

View File

@@ -1,9 +1,11 @@
# Published Evidence Context
This contract defines the optional `source/evidence-context` artifact emitted
by the production JSON output. Its configuration is owned by
[Configuration](../config.md#module-bindings-and-validators); its logical-file
discovery is owned by [Published JSON Output](json-output.md).
by the production JSON output. It is a selected source-unit excerpt for
convenient reading alongside normalized lane artifacts; it is not a second
citation or provenance model. Its configuration is owned by
[Configuration](../config.md#module-bindings-and-validators), and its
logical-file discovery is owned by [Published JSON Output](json-output.md).
## Identity And Discovery
@@ -26,91 +28,80 @@ its absence means evidence publication was not enabled for that bundle.
## Payload
The v1 payload is a JSON object with required `source_id`, `source_digest`,
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
`contexts` are always arrays; an enabled configuration with no accepted direct
evidence publishes `contexts: []`.
The v1 payload is a top-level JSON array of generic source units. There is no
wrapper, source-level metadata, context grouping, lane identifier, or evidence
reference in the payload. An enabled configuration with no contributing
accepted evidence publishes `[]`.
```json
{
"source_id": "session-alpha",
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
"window_units": 1,
"selected_lanes": ["npc_registry", "spells"],
"contexts": [
{
"context_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 20
},
"evidence_refs": [
{
"lane_id": "spells",
"source_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
}
],
"units": [
{
"id": 10,
"kind": "transcript_segment",
"text": "Aria casts Cure Wounds.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
},
{
"id": 20,
"kind": "transcript_segment",
"text": "The party regroups.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 20,
"end_unit_id": 20
}
}
]
[
{
"id": 10,
"kind": "transcript_segment",
"text": "Aria casts Cure Wounds.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
]
}
},
{
"id": 20,
"kind": "transcript_segment",
"text": "The party regroups.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 20,
"end_unit_id": 20
}
}
]
```
Each context requires a `context_ref` object and `evidence_refs` and `units`
arrays. `context_ref` identifies the first and last included unit. Each
evidence entry contains a selected `lane_id` and an original `source_ref`. A
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
unknown fields; unit metadata may contain application-defined JSON values.
Each source unit has required `id`, `kind`, `text`, and self `ref` fields.
`ref` contains `source_id`, `start_unit_id`, and `end_unit_id`, and both unit
endpoints identify that unit's `id`. A unit may also contain source-owned
`metadata`, an open-ended JSON object. Fixed unit and reference fields are
strict: consumers must reject unknown fixed fields, malformed units, invalid
self-references, units whose `source_id` differs from other units in the same
excerpt, and a payload that is not the array described here.
## Citations And Context
The excerpt preserves each selected unit exactly as represented by the
validated generic source document. It does not add evidence-context-specific
annotations or reshape source-owned metadata.
`evidence_refs` are the authoritative citations. They identify the direct
references emitted by accepted normalized artifacts. `context_ref` and the
units collection include those cited units plus nearby source units selected by
the configured window. They are explanatory context, not widened citations.
## Selection And Citations
Only accepted outputs from the configured lane allowlist contribute. Rejected,
failed, absent, and lane-filtered outputs do not contribute. The artifact never
contains raw input bytes, prompts, model responses, auxiliary reference
content, credentials, or filesystem paths.
The framework obtains direct source references only through typed evidence
projections of accepted normalized artifacts in the configured lane allowlist.
It validates each reference against the current source document, expands its
range by `window_units` source-unit positions on each side, clamps at document
boundaries, and takes the union of all expanded ranges. The output contains
each selected source unit once in source-document position order, regardless
of numeric unit IDs. Repeated references, overlapping windows, and citations
from multiple lanes do not duplicate a unit. Rejected, failed, absent,
inactive, and unselected lanes contribute nothing.
## Ordering And Compatibility
Normalized lane artifacts remain authoritative for citations and for which lane
cited a range. The excerpt has no lane attribution and must not be used to
reconstruct it. Its included nearby units provide reading context only; they
do not widen any citation in a lane artifact.
The selected lane allowlist is lexical. Contexts and units are in source
document position order, not numeric unit-ID order. Direct evidence entries
are deterministically ordered by lane and source reference. Overlapping or
contiguous windows merge, and each source unit appears at most once in the
resulting contexts.
The excerpt contains at most every generic source unit once. It can therefore
equal the complete generic source document when coverage is broad or the
window is large. No byte-, token-, or compression-size guarantee is made, and
the framework does not truncate the excerpt to meet an arbitrary size limit.
## Consumer Responsibilities And Data Handling
The artifact is additive to the JSON bundle and is not a lane payload,
normalized-output count, checkpoint, or generated reference. Consumers that
do not need it must tolerate the absent optional descriptor. Consumers that do
use it should preserve the artifact and its schema identity with the run
provenance, and should treat its source text and metadata as sensitive durable
content.
do not need it must tolerate an absent descriptor. Consumers that do use it
should validate the descriptor and payload before use, retain the artifact with
its schema identity when needed for a run record, and read citations from the
corresponding normalized lane artifacts.
The excerpt contains source-unit text and source-owned metadata and is durable
output. Treat it as sensitive source content, apply appropriate access controls
and retention, and do not assume its selected form is materially smaller or
less sensitive than the original input.

View File

@@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo
## Bundle Layout
All paths below are logical, relative, slash-separated bundle paths. The
encoder always emits the first four JSON files below and adds lane or
encoder always emits the first five JSON files below and adds lane or
pipeline-wide artifact files when their corresponding artifacts are available:
A subprocess caller first obtains the physical bundle root from the
@@ -21,10 +21,11 @@ root for the logical discovery described here.
| `index.json` | Entry point that names the other published files and lane payloads. |
| `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. |
| `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. |
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
JSON files are pretty-printed with a trailing newline. Lane payloads are
accepted only when their media type is `application/json`.
@@ -39,7 +40,8 @@ normalized lanes has this valid minimal index:
"manifest_file": "manifest.json",
"output_files": [],
"rejected_file": "rejected.json",
"warnings_file": "warnings.json"
"warnings_file": "warnings.json",
"diagnostics_file": "diagnostics.json"
}
```
@@ -49,6 +51,7 @@ normalized lanes has this valid minimal index:
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
| `rejected_file` | Yes | Always `rejected.json`. |
| `warnings_file` | Yes | Always `warnings.json`. |
| `diagnostics_file` | Yes | Always `diagnostics.json`. |
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
@@ -92,7 +95,7 @@ group into the following externally observable summaries:
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
| Source and references | `source_digests`, `references` |
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
| Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
`references` records provenance such as the target, slot, origin, digest,
@@ -102,6 +105,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
provenance for the plan used by this run; cache records, debug artifacts, and
other operational state are not published as bundle files.
Each `validation_summaries` entry is a bounded outcome for one producer result.
It has required `status`, `producer_attempt_count`, and `terminal_action`;
the stage and affected step, lane, module, or chunk identity are present when
applicable. `status` is `complete`, `rejected`, or `incomplete`.
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
configured validator order and omit later duplicates. Entries contain no raw
candidate response, correction guidance, validator diagnostic message, or
artifact payload. The same shape may appear as `validation` on an affected
rejection entry.
When present, `metadata.session_id` is the effective non-secret routing
correlation identifier used for the run. It can be visible to providers and is
not a substitute for a cache or checkpoint identity. Its generation and
@@ -122,16 +135,57 @@ These values describe observed execution; they are not a backend-registration
interface. Entries that differ by backend or effective reasoning remain
distinct even when their profile, provider, and model are otherwise equal.
## Rejections And Warnings
## Rejections, Warnings, And Diagnostics
`rejected.json` is always an object with a `rejected` array. Each entry has
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` are present only when applicable.
`diagnostic_artifact_path` are present only when applicable. An entry may also
contain the bounded `validation` summary described above; the existing singular
validator and reason fields remain the first configured rejection for
compatibility.
`warnings.json` is always an object with a `warnings` array. Each warning has
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
there is nothing to report.
`warnings.json` is always the `notarius.warnings.v2` envelope:
```json
{
"schema_version": "notarius.warnings.v2",
"group_count": 0,
"occurrence_count": 0,
"groups": []
}
```
It contains only process warnings. `group_count` is exact, and
`occurrence_count` is the exact sum of its group occurrence counts.
`diagnostics.json` is always the `notarius.diagnostics.v1` envelope:
```json
{
"schema_version": "notarius.diagnostics.v1",
"group_count": 0,
"occurrence_count": 0,
"truncated": false,
"unrepresented_occurrence_count": 0,
"groups": []
}
```
It contains only advisory and observation groups. `group_count` counts groups
represented in `groups`; `occurrence_count` includes both represented and
unrepresented occurrences. When `truncated` is true,
`unrepresented_occurrence_count` is the exact number omitted from group
representation.
Each group has `disposition`, `category`, `reason_code`, framework-owned
`origin`, exact `occurrence_count`, bounded `samples`, and
`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk
ID and zero-based chunk index when applicable. A group retains at most three
distinct samples. The framework fails rather than truncating actionable
warnings beyond 128 groups; it represents at most 256 advisory/observation
groups and records further occurrences through the diagnostic truncation
fields above.
## Compatibility

View File

@@ -1,11 +1,11 @@
# PromptKit Integration
Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
[`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
owns prompt, profile, and schema file contracts.
## Supported Boundary
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
- construct an `Engine` with filesystem-backed prompt, schema, and optional
operator and application-fallback profile sources;
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
variables, a direct session ID, prompt identity, and profile selection, then
variables, a direct session ID, prompt identity, profile selection, and
optional appended rendered messages, then
record credential-redacted details and run that exact execution;
- return rendered debug material, validated structured output, selected
profile, backend, effective model metadata, and token usage;
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
admission exhaustion through `ErrCapacityExceeded`.
The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
owns the registration and backend-capacity contract.
For one completion, the adapter calls `PrepareExecution`, takes a
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
field, which is authoritative for provider session behavior. It also retains
the same value as the `session_id` prompt variable for maintained prompt
compatibility. The generated identifier is 76 ASCII characters, within
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret
PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
correlation identifiers and may be exposed to providers and provider
observability. The CLI contract owns generation and override behavior.
@@ -84,7 +85,39 @@ configuration and deployment workflow are defined in
[Configuration](../config.md#promptkit-profiles) and
[Operations](../operations.md#promptkit-profile-deployment).
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403.
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
`assistant` text-chat roles after normalizing case and surrounding whitespace.
Maintained Notarius prompt definitions use only `system` and `user`.
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
exactly two messages in order: the latest validated producer response with
role `assistant`, then deterministic validation guidance with role `user`.
It never exposes a general caller-selected role API, accumulates earlier
correction turns, or changes the ordinary prompt prefix. Ordinary requests
leave appended messages unset.
PromptKit preserves supplied content but does not own Notarius's correction
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
1,114,112 bytes) before preparing the request. The transport-neutral
application contract owns defensive copying and these limits. Default request
and terminal summaries retain only safe counts, digests, identities, and usage;
complete appended messages remain limited to the explicitly requested detailed
debug trace.
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
profile catalogs from independently versioned transitive modules. Notarius
does not import or register either catalog; PromptKit retains catalog source,
identity, precedence, credential, and capacity ownership.
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter,
@@ -97,6 +130,10 @@ pinned upstream documentation.
module assets, maps its transport-neutral completion contract, prepares and
executes requests, validates output, records provenance, captures debug
material, redacts errors, and preserves timeout ownership.
PromptKit provider error details do not cross the ordinary completion boundary.
Notarius exposes a provider-neutral generation category and optional status;
redacted provider details are retained only in requested debug material.
[D&D Module Internals](../internal/dnd.md) owns the embedded
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
@@ -105,4 +142,7 @@ the conventional local backend.
PromptKit API or format changes outside this boundary are not implicitly
supported. Updating the pinned version requires reviewing the adapter and
profile/configuration contracts against the upstream documentation.
profile/configuration contracts against the upstream documentation. Maintained
production prompts use PromptKit's bounded structural-repair contract; their
current declaration is one additional repair attempt. Notarius retains the
transport-neutral boundary and does not expose PromptKit types to modules.

View File

@@ -9,36 +9,58 @@ Command syntax, streams, and exit statuses are defined in the
## Schema
The current schema version is `notarius.run-result.v1`.
The current schema version is `notarius.run-result.v2`.
| Field | Required | Meaning |
| --- | --- | --- |
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. |
| `schema_version` | Yes | Exactly `notarius.run-result.v2`. |
| `run_id` | Yes | The finalized Notarius run identifier. |
| `pipeline_id` | Yes | The effective pipeline identifier. |
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
| `normalized_output_count` | Yes | Number of final normalized outputs. |
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
| `warning_count` | Yes | Number of final run warnings. |
| `warning_group_count` | Yes | Exact number of actionable warning groups. |
| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. |
| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. |
| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. |
| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. |
| `validation_status` | Yes | The final run manifest validation status. |
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
For the production `json` output module, `index_file` is present only when the
completed run returned exactly one logical output file named `index.json`.
For another output module, its absence does not indicate a failed run.
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
means one or more otherwise accepted results advanced under validator-failure
`warn_continue`.
```json
{
"schema_version": "notarius.run-result.v1",
"schema_version": "notarius.run-result.v2",
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
"pipeline_id": "dnd-session",
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
"index_file": "index.json",
"normalized_output_count": 6,
"rejected_output_count": 2,
"warning_count": 1,
"validation_status": "rejected"
"warning_group_count": 1,
"warning_occurrence_count": 2,
"diagnostic_group_count": 3,
"diagnostic_occurrence_count": 5,
"diagnostics_truncated": false,
"validation_status": "incomplete",
"validation_summaries": [
{
"stage": "extract",
"lane_id": "spells",
"status": "incomplete",
"incomplete_validators": ["dnd/spells/source_refs"],
"producer_attempt_count": 1,
"terminal_action": "warn_continue"
}
]
}
```
@@ -48,8 +70,11 @@ For another output module, its absence does not indicate a failed run.
paths. They identify the paths used by Notarius and do not resolve symlinks.
`output_directory` is the run-specific bundle, not the configured output root.
The receipt is a summary and discovery document. It does not contain lane
descriptors, payloads, manifest data, rejections, warnings, or file contents.
The receipt is a summary and discovery document. Its optional validation
summaries contain only stable status, identity, validator names, reason codes,
attempt counts, and terminal actions. It does not contain lane descriptors,
payloads, manifest payloads, rejection messages, warnings, raw model responses,
correction guidance, or file contents.
For the production JSON output, resolve `index_file` beneath
`output_directory`, reject path escapes, and use the
[Published JSON Output contract](json-output.md) to discover logical files and

View File

@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
## Dispatch And Configuration Handoff
The root dispatcher handles help, configuration validation, pipeline listing,
and a pipeline run. It normalizes injectable options before dispatch so that a
missing production dependency fails as a command error rather than reaching
execution.
The root dispatcher handles help, version reporting, configuration validation,
pipeline listing, and a pipeline run. Version reporting resolves build
information through `internal/buildinfo` before production composition, so the
diagnostic remains available without configuration or runtime collaborators.
The public syntax, streams, exit classes, and version semantics are defined by
the [CLI reference](../cli.md). Other root commands normalize injectable
options before dispatch so that a missing production dependency fails as a
command error rather than reaching execution.
Commands that need configuration use one shared loader. The CLI discovers the
file, parses it through **internal/core/config**, starts from defaults, applies

View File

@@ -74,10 +74,19 @@ profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by
[Configuration](../config.md#pipelines).
The resolver retains configured `validation_policy` overrides and derives one
detached concrete terminal policy for the chunk producer and every lane's
extract, merge, and normalize producers. That field-by-field inheritance is
complete before preparation, and the effective values contribute to pipeline
and checkpoint identity; execution does not interpret configuration defaults.
The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting
**EffectiveConfig** retains the selected ID, requested selection and reference
options, and returns the fixed ordered pipeline shape. Positive validator retry
budgets require an LLM-backed selected validator; deterministic validators are
rejected during resolution. Eligible LLM-backed producer specifications also
contribute their declared correction protocol to the resolved metadata. The
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
changes, a clone of the input configuration, and the resolved pipeline.
Callers may therefore retain or modify their input slices and maps without
changing the resolved result, and later consumers cannot mutate the original
@@ -94,8 +103,8 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
The framework assigns the resolved pipeline a deterministic SHA-256 digest
after defaults, lane selection, module bindings, reference bindings, validator
chains, effective LLM profiles, and artifact schema identity have been
resolved. The digest excludes
chains, selected correction protocols, effective LLM profiles, and artifact
schema identity have been resolved. The digest excludes
its own stored value. It identifies resolved composition rather than raw YAML
bytes, a debug payload, or all runtime state. The CLI records it as invocation
provenance before execution; cache and checkpoint identity have additional

View File

@@ -33,7 +33,9 @@ typed builder. Scene chunking, every extractor, and NPC, location, and item-regi
normalization are registered as `llm_backed`; the remaining current D&D mergers
and normalizers are `deterministic`. The metadata is available to catalog inspection and
resolved-pipeline debug data and determines which selected bindings inherit the
pipeline profile. Configuration remains the canonical owner of the exact keys,
pipeline profile. The registry normalizers use `single_response_v1`, forwarding
corrections to their reconciliation completion and retaining the accepted raw
proposal only as an owned model candidate. Configuration remains the canonical owner of the exact keys,
profile precedence, and validator order.
Private structured-LLM response schemas are deliberately minimal. They reject
@@ -42,6 +44,13 @@ unknown fields, while preserving semantic candidates for deterministic
validation. Do not promote a private response envelope into a durable schema;
the contracts above define durable data.
A D&D producer that declares `single_response_v1` forwards any supplied
semantic correction to its structured completion and returns an owned copy of
that completion's exact validated raw response as its model candidate. It does
not serialize normalized artifacts to create that candidate, so deterministic
identity, evidence, warning, and durable-schema behavior remains separate from
the model transport material.
## Prompt Construction
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
@@ -84,14 +93,15 @@ replace it with a complete profile of the same ID from the configured PromptKit
source. Deployment profile selection is documented in
[Configuration](../config.md#promptkit-profiles).
The transcript assets have distinct consumers. Scene chunking consumes the
complete-session `common-dnd-transcript-full.md`; extraction prompts consume
the current-chunk `common-dnd-transcript-chunk.md`; and NPC, location, and item
normalization consume `common-dnd-transcript-windows.md` alongside their
candidate collections. Player, party, glossary, and compatible campaign
references provide disambiguating context only when declared by the active
prompt; they never establish evidence. Reference material is canonically
ordered before rendering so equivalent inputs remain stable.
The D&D transcript assets have distinct consumers. Scene chunking consumes the
complete-session `common-dnd-transcript-full.md`, while extraction prompts
consume the current-chunk `common-dnd-transcript-chunk.md`. NPC, location, and
item normalization instead mount the generic semantic-reconciliation
candidate and transcript-window presentation assets. Player, party, glossary,
and compatible campaign references provide disambiguating context only when
declared by the active prompt; they never establish evidence. Reference
material is canonically ordered before rendering so equivalent inputs remain
stable.
Extraction prompts render the common system and identity messages first, then
cached campaign references and the cached chunk transcript. Evidence policy and
@@ -101,10 +111,11 @@ reusable extraction prefix identical while preserving the lane-specific suffix.
Scene chunking intentionally uses a different order: system, cached campaign
references, uncached module instructions, then the final ephemeral full
transcript. Entity normalization also has its own order: system, uncached
module instructions, ephemeral reconciliation policy, uncached candidates, and
final ephemeral transcript windows. These orders and cache controls are prompt
behavior; change them only through the owning manifest and prompt declaration.
transcript. Entity normalization also has its own order: D&D system, mandatory
generic protocol, ephemeral domain semantic instructions, generic candidate
presentation, and final ephemeral generic transcript windows. These orders and
cache controls are prompt behavior; change them only through the owning
manifest and prompt declaration.
## Evidence, Candidates, And Normalization
@@ -123,6 +134,13 @@ relatedness validators report advisory evidence concerns. The configured order
is documented in
[Configuration](../config.md#production-validator-keys-and-default-chains).
Every D&D rejection describes the correction in transcript-grounded domain
terms, using contextual names, artifact fields, and source segment ranges when
useful. The guidance must not ask the model to reproduce durable entity IDs,
hashes, validator module keys, or reason codes. Those identifiers remain in
ordinary validation provenance; only the actionable semantic guidance is
eligible for the correction prompt.
Enemy-event extraction additionally rejects a second `engaged` observation for
the same comparison identity within one scene-scoped result. Normalization may
combine results from distinct scenes, so it intentionally does not apply that
@@ -131,15 +149,49 @@ rule. Configuration owns the exact validator key and chain position.
Normalizers are deterministic for spells, combat turns, item occurrences, NPC
occurrences, scene descriptions, enemy events, and location occurrences. They
canonicalize display values and evidence, use source-document order for stable
output, and issue bounded warnings for changes or collapsed duplicates. NPC,
output, and emit bounded normalization observations for changes or collapsed duplicates. NPC,
item, and location registry normalizers are intentional exceptions: each first
produces a deterministic candidate set, then may use a bounded structured-LLM
proposal to reconcile identity groups. The proposal selects supplied
descriptors—names with their candidate source references—not durable IDs.
Request-local candidate keys may support resolution internally, but are never
included in model input or output. Colliding descriptors are ineligible, and
invalid or unusable proposals retain the deterministic result with retry or
fallback diagnostics; the model does not directly replace durable records.
proposal to reconcile identity groups.
## Semantic Registry Reconciliation
The three registry normalizers instantiate the domain-neutral
`internal/framework/semanticreconcile` engine with default bounds. Each
eligible candidate receives a contiguous, one-based `candidate_id` for that
request. The model sees that handle, the candidate label and source-free
evidence ranges, plus bounded transcript windows; it returns only duplicate
groups of supplied handles and one supplied canonical handle per group. It
never returns names, evidence, durable IDs, or replacement records. Identical
labels and evidence remain independently selectable because their handles are
distinct.
The generic core owns the mandatory handle protocol, candidate and transcript
presentation, the private response schema, source-reference validation,
candidate and combined-material limits, structured completion, proposal
assessment, stable group ordering, and typed plan-application mechanics. The
D&D prompt contributes its system message and registry-specific semantic
instructions. The generic registrar registers the shared prompt and schema;
the D&D registrar registers each consuming prompt and the fallback profile.
Fewer than two eligible candidates skips the LLM without a semantic warning.
An exceeded bound also skips the call and preserves the deterministic
preprocessed registry, adding the registry's bounded fallback warning. Invalid
structured output or discarded proposal groups use the normalizer's existing
retry contract; retry exhaustion preserves the safe deterministic or
partially applied result and emits its bounded fallback warning. Provider,
transport, cancellation, and context-material failures remain execution
errors.
Application remains typed and registry-owned. All three policies select the
canonical member's normalized display name, union member evidence in source
order, preserve ungrouped records, and derive durable identity only after
consolidation. NPC IDs derive from the final name. Item IDs also derive from
the final name, and a typed guard prevents currency aliases from crossing
denominations or mixing currency with non-currency records. Location IDs
derive from the final name and final evidence, preserving same-name,
parent/child, and distinct physical-place identities. Registry warning scopes,
reason codes, and postconditions remain outside the generic core.
## Generated References And Grounding

View File

@@ -42,6 +42,22 @@ observability. The adapter returns PromptKits validated raw bytes rather than
re-encoding the decoded target. An empty optional material is represented as
one space so its named input is retained by PromptKit.
When a request includes semantic correction material, the adapter validates and
defensively copies it before preparation, then appends exactly two messages
after the ordinarily rendered prompt: the prior response as an assistant
message and the correction guidance as a user message. Requests without a
correction do not add messages or introduce caller roles. Ordinary request
summaries record correction byte counts and digests only; complete messages are
available solely in an explicitly requested debug trace.
The adapter leaves the ordinary rendered message prefix, named inputs,
variables, session, profile, execution overrides, prepared-execution path, and
PromptKit repair policy unchanged for a corrected request. It never imports a
PromptKit message type into a module or pipeline contract. PromptKit reports
actual structural repair count and cumulative token usage per completion; the
pipeline's safe terminal debug record projects those values without copying
message content.
Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer
@@ -75,7 +91,7 @@ backend membership as runtime without performing generation. Fallback assets
are mounted only when at least one source is registered. The production D&D
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
prompts select that logical ID by default. PromptKit owns source precedence and
profile parsing: an operator-provided matching profile takes precedence over a
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
fallback profile without Notarius merging either document.
When the registration is absent, a profile selecting `backend: local` fails
inspection instead of falling back to a built-in or endpoint-only target.
@@ -142,12 +158,28 @@ arrangement and its data-only boundary are defined by
[ADR-0011](../adr/0011-centralize-llm-assets.md), rather than by this runtime
guide.
The generic registrar is the sole production registration owner for the
semantic-reconciliation default prompt and private response schema. The
domain-neutral reconciliation package also exposes only its mandatory protocol
and candidate/transcript presentation files for domain prompt manifests. D&D
registry normalizers mount those files while retaining ownership and hashing
of their D&D system message, semantic instructions, and complete prompt
declaration. The response schema is therefore registered once even though
several typed normalizers select it.
Mounted prompt assets determine a module's fingerprint. The fingerprint hashes
only the module and shared files explicitly selected by its manifest, so an
unrelated asset does not invalidate a checkpoint. Schema loaders validate JSON,
attach identity and digest metadata, make defensive copies, and expose
diagnostics without raw schema bytes.
Semantic-reconciliation normalizers extend this identity with the shared
response-schema digest, framework policy version, and complete limit-policy
digest. Their manifest metadata records the same content-free prompt, schema,
policy, and limit identities together with domain identity and normalization
policies. Request-local handles, source material, proposal content, and raw
asset bytes are not checkpoint metadata.
Private response schemas validate a model transport envelope. They are not the
durable artifact schema and should not be documented as an external wire
contract. Durable formats and compatibility rules remain in the
@@ -194,13 +226,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
the pipeline's existing binding attempt policy sees the operational error and
decides whether to rerun the complete operation.
Prompt-declared repair is executed within PromptKits structured-output flow.
The current production D&D prompt manifests set repair attempts to zero. That
setting does not replace pipeline retry behavior: a bindings configured retry
count reruns its stage attempt after an error or rejection, and an exhausted
rejection is a recorded output rather than a provider error. The pipeline owns
attempt lifecycle, validation chains, and retry diagnostics; see
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
PromptKit executes structural repair within its structured-output flow. The
maintained production prompt manifests declare one additional repair attempt.
When a resolved binding supplies a repair value, the adapter inspects the
prompt, copies its complete output contract, changes only the repair limit, and
passes that complete replacement contract to PromptKit. This preserves the
prompt's output format, validation mode, schema, and provider structured-output
settings.
A successful repair is an ordinary successful completion, not a warning. The
adapter reports PromptKit's actual repair count and its cumulative usage
directly, without adding the initial and corrective counts again. Debug prompt
material records the configured complete contract; debug response material
records the repaired response and actual validation result. If the repair
budget is exhausted, the adapter retains the final raw bytes and debug material
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
or corrective call remain provider-neutral operational errors with the same
redaction boundary.
Structural repair does not replace pipeline retry behavior: a binding's
configured retry count reruns its complete stage attempt after an operational
or structural error, module-requested retry, or actionable semantic rejection.
The pipeline owns attempt lifecycle, validation chains, and retry diagnostics;
see [Pipeline Internals](pipeline.md#validation-retries-and-output) and the
[binding reference](../config.md#module-bindings-and-validators).
## Timeout Ownership
@@ -231,6 +279,13 @@ surfaced; when the completion already failed, its call error remains the
result. Debug-bundle location, retention, and handling are operational concerns
documented in [Operations](../operations.md#debug-bundles).
The attempt-terminal summary is a separate safe trace record: it contains
attempt kinds, validator outcome counts and reason codes, effective policy,
terminal action, and repair/usage references. It excludes raw assistant
responses and correction text. Those values can appear only in the explicitly
requested detailed prompt and response artifacts, which require sensitive-data
handling.
Run manifests receive selected profile summaries, including optional effective
backend and reasoning provenance, and component identities—not prompt, schema,
source, reference, or response content. The published field semantics belong
@@ -240,6 +295,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
available to other runtime collaborators; it does not make prompt or response
contents safe for general logging.
Generation failures expose an application-owned category and optional HTTP
status. Provider code, type, and message remain debug-only, after redaction.
## Failure Boundaries
- Construction fails for missing asset registries, mutually exclusive profile

View File

@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
binding-specific profile on a deterministic module. The user-facing precedence
contract belongs in [Configuration](../config.md#pipelines).
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
declare correction protocol `single_response_v1`. That declaration is a
promise that the implementation accepts one attempt-local semantic correction
and returns an owned copy of the exact one model response that directly
controlled the candidate. It must forward correction only to its structured
completion request; it must not manufacture prior-response material by
serializing a normalized artifact or expose opaque application IDs. Input,
output, validator, and deterministic specs cannot declare the protocol.
Implementations that accept options must provide both an option validator and
a builder. The validator is used while resolving configuration; the builder
decodes the same options and constructs the implementation from the prepared
@@ -36,20 +45,38 @@ they need, register each leaf implementation, and add any family-owned assets
or default validator chains. They return contextual errors so production
composition fails at startup rather than at the first run.
A validator that returns a completed rejection must supply two separate
bounded values: a stable `ReasonCode` for provenance and actionable
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
and the constraints on one complete corrected replacement. It must not contain
validator keys, diagnostic paths, opaque application IDs, or other internal
identifiers. An operator-facing `Message` may explain the same event, but the
framework never copies it into a model request. Missing or invalid guidance is
a validator contract failure.
An artifact family can register an optional typed evidence projector alongside
its codec. The projector returns defensive copies of the artifact's direct
generic source references and must use the codec's exact Go type. It does not
interpret surrounding context or publish files; the pipeline validates the
capability during preparation and the output boundary owns publication. See
the [Published Evidence Context contract](../integrations/evidence-context.md)
for the durable result.
for the durable source-unit excerpt. Lane artifacts retain citation and lane
provenance; the framework does not add either to that published excerpt.
An artifact family is broader than a module: it owns the cohesive domain
feature across its artifact type, codec, stage modules, validators, prompt
policy, schemas, identity helpers, and reference projections. An extractor and
normalizer in one artifact family remain independently registered modules in
their respective pipeline stages. This ownership vocabulary does not create a
new registry or change the fixed pipeline.
## Production Composition
Production composition is intentionally split by family:
- The generic registrar provides the unit chunker, generic JSON validators,
and JSON output encoder.
JSON output encoder, and shared semantic-reconciliation prompt and response
schema assets.
- The Seriatim registrar provides the transcript input adapter. Its external
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
@@ -60,6 +87,42 @@ The CLI owns the composition that invokes these registrars. A module package
may register its own family but must not assemble the CLI or make framework
packages depend on production extensions.
## Semantic Reconciliation
`internal/framework/semanticreconcile` is a domain-neutral strategy used by a
typed normalize module; it is not itself a selectable stage module. A
source-backed artifact-family normalizer projects its deterministic records
into contextual candidates and owned typed record envelopes, supplies its
chosen prompt identity and resolved LLM profile, and constructs an engine with
explicit limits. The core filters invalid evidence, assigns contiguous
request-local integer handles, renders bounded candidate and transcript
materials, invokes the structured-completion boundary, and assesses the
returned duplicate groups into a stable non-overlapping plan.
The normalizer then applies that plan through a typed `ApplicationPolicy`. The
core preserves ungrouped records, contribution order, and provenance while the
artifact family owns group guards, field and evidence consolidation, durable
ID derivation, retry and fallback presentation, classified diagnostics, and postconditions.
Request-local handles do not enter the typed value or durable artifact. Fewer
than two eligible candidates skips model invocation; exceeding a candidate or
combined-material bound preserves the deterministic result under the family's
fallback policy. Provider, transport, cancellation, and context-construction
failures remain execution errors.
When the engine actually makes a proposal call, its typed result carries the
owned exact proposal response under the same correction contract as other
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
model candidate, so a later rejection applies terminal policy without spending
an ineffective semantic retry.
The core supplies a conservative generic prompt and the single private
response schema. A domain prompt may substitute its semantic instructions but
mounts the core-owned protocol and candidate/transcript presentation assets.
Prompt, schema, policy, and limit identities participate in manifest metadata
and checkpoint fingerprints. The generic registrar owns production
registration of those shared assets; a consuming domain registrar owns only
its domain prompt.
## Adding Or Changing A Module
1. Choose the pipeline stage and the typed artifact boundary. Put external
@@ -72,9 +135,13 @@ packages depend on production extensions.
3. Implement strict option decoding, construction, and the typed stage
interface. Preserve caller ownership: do not retain mutable request data
and return defensive copies where an implementation exposes stored data.
If declaring correction capability, forward the request correction and
retain only the exact validated response that controlled the result.
4. Register the module through its typed registry helper and add it to the
owning family registrar. Add a default validator chain only when that
family owns the behavior; otherwise require an explicit compatible chain.
Every rejection path in a validator must provide actionable correction
guidance while retaining its stable internal reason code.
5. Update the selectable-key and chain reference in
[Configuration](../config.md#production-module-keys), the applicable
integration contract, and focused tests. Keep the configuration document

View File

@@ -25,10 +25,12 @@ physical state roots.
| Area | Implemented owners | Responsibility |
| --- | --- | --- |
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
| Semantic reconciliation | **internal/framework/semanticreconcile** | Bounded source-backed candidate preparation, request-local handle proposals, deterministic assessment, typed plan application, and reconciliation identity metadata; see [Module Internals](modules.md#semantic-reconciliation) and [D&D Module Internals](dnd.md#semantic-registry-reconciliation). |
| Embedded LLM content | **assets** | Read-only centralized LLM-facing content, scoped by its consuming package; see [LLM Runtime](llm.md#prompt-and-schema-assets) and [D&D Module Internals](dnd.md#prompt-construction). |
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
@@ -49,8 +51,9 @@ the CLI composition boundary.
composition, and path safety.
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
profiles, and secret handling.
- [Module Internals](modules.md): generic extension registration, module
construction, validation, and reference mechanics.
- [Module Internals](modules.md): generic extension registration, artifact
families, module construction, semantic reconciliation, validation, and
reference mechanics.
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
reference projections, and lane-specific exceptions. Durable D&D and
Seriatim data shapes remain in the [integration contracts](../integrations/).

View File

@@ -12,7 +12,7 @@ own durable output shapes. Concrete production extensions are covered by
The pipeline framework accepts a resolved composition, registries, shared
dependencies, input bytes, a supplied prompt session, and state/debug
collaborators. It returns logical output files, normalized artifacts, recorded
rejections and warnings, manifest provenance, and checkpoint decisions. The
rejections, grouped diagnostics, manifest provenance, and checkpoint decisions. The
CLI owns process arguments, configuration discovery, session resolution,
physical roots, and placement of returned output files.
@@ -32,13 +32,26 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
It normalizes the pipeline and lane identities, applies stage defaults, selects
requested lanes where that is supported, resolves validator chains, checks
module capabilities and typed artifact compatibility, validates options, and
assigns a deterministic resolved-composition digest. The resolved pipeline
contains bindings and declared reference targets, not external reference bytes.
assigns a deterministic resolved-composition digest. A correction protocol is
selected from each eligible LLM-backed producer specification and becomes part
of that resolved identity; only `single_response_v1` is currently supported.
Preparation rejects an LLM-backed producer that combines a non-empty validator
chain with positive producer retries unless it declares that protocol. Producers
without validators or without retries remain valid without correction support.
The resolved pipeline contains bindings and declared reference targets, not
external reference bytes.
After selection, the resolver applies command, binding, and pipeline profile
precedence to LLM-backed bindings and validators only; prompt defaults remain
an empty resolved binding profile. Deterministic bindings remain profile-free.
These effective values are part of the digest, so execution and checkpoint
consumers do not repeat profile inheritance.
an empty resolved binding profile. It resolves structural output repair
separately: a binding's `structured_output_repair_attempts` value wins, then a
pipeline value applies to LLM-backed bindings and validators, and omission
leaves the prompt-owned policy intact. An explicit repair value on a
deterministic binding is rejected. Resolved bindings own copied repair values,
and these effective values are part of the digest, so execution and checkpoint
consumers do not repeat profile inheritance or configuration resolution.
Each LLM request receives its own copy of that resolved value. PromptKit spends
it only for structural correction inside one completion; the runner's binding
retry policy remains the separate outer budget for complete stage attempts.
Configuration resolution supplies the selected profile and catalog; see
[Configuration Internals](configuration.md).
@@ -52,9 +65,11 @@ remains declared but has no bytes until its producing step completes.
Preparation is the construction boundary. It validates the resolved shape and
registry set, clones the resolved data, then constructs the input adapter,
chunker, stage-local validators, every typed lane, and output encoder. Each
registered builder receives its own cloned build request immediately before its
module-owned code runs. Preparation also collects stable checkpoint
chunker, stage-local validators, every typed lane, and output encoder. The
prepared producer metadata preserves each selected correction protocol, and
the resolved digest carrying that metadata participates in checkpoint identity.
Each registered builder receives its own cloned build request immediately
before its module-owned code runs. Preparation also collects stable checkpoint
fingerprints. Missing registrations, incompatible typed entries, nil
implementations, and constructor failures are reported before source parsing
or any stage operation begins.
@@ -121,17 +136,48 @@ their target: chunks, codec-decoded typed candidates, or serialized codec
bytes. Each typed validator receives a newly decoded value from the one
candidate serialization for that attempt, while serialized validators receive
separately owned representation bytes and schema metadata. They may approve,
approve with warnings, reject, or fail. A rejection is an ordinary pipeline
result; a validator error is a framework error.
approve with warnings, reject, fail, or be skipped when a runtime prerequisite
is unavailable. The shared executor settles every configured validator in
order. A failed LLM-backed validator retries only itself against the same
immutable candidate; it does not regenerate the producer or alter the
validator request. Rejections stop that validator, while other configured
validators still run. The executor retains ordered results, bounded
deduplicated correction guidance from rejections, and only the final exhausted
failure outcome for each validator. The correction builder keeps first
occurrence order, omits internal reason codes, validator names, and operator
messages, and requests one complete replacement. Missing guidance or an
oversized aggregate is a framework contract error; guidance is never inferred
or truncated.
The runner applies the binding's retry policy around a stage operation and its
complete validation chain. It preserves warnings only from the final accepted
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
directives consume this same budget and validate any final safe fallback through
the normalizer chain.
complete validation chain. It preserves terminal diagnostics only from the final accepted
or rejected attempt, plus one fixed validation-incomplete warning per validator whose execution
budget was exhausted under `warn_continue`. Cancellation stops retries.
Normalizer-specific retry directives consume this same budget and validate any
final safe fallback through the normalizer chain.
The artifact-neutral producer-attempt state machine owns that shared budget,
attempt provenance, semantic-correction material, and terminal-policy
selection. It accepts producer and complete-validation closures, so artifact
materialization, cache handling, checkpoints, and debug output stay at the
operation boundary. It distinguishes operational, structural, module-requested,
and semantic retries. A semantic retry is available only for a valid latest
`single_response_v1` candidate; a deterministic or no-model rejection instead
settles the semantic policy immediately. Structural-output errors alone use the
structural policy, and validation failure without rejection settles the
validator-failure policy without regenerating the producer.
Chunk planning uses this state machine for generated plans. A rejected or
validation-incomplete automatic cache hit is not model material and therefore
falls through to a fresh initial generation at producer attempt one; it neither
receives a correction, consumes retry budget, promotes cached-candidate
diagnostics, nor overwrites the stored record. An incomplete cache validation
under `fail_run` terminates instead. Only a newly generated, completely
validated plan is published to the chunk-plan store. Rejected plans never
advance, and validation-incomplete plans remain unpublishable.
After terminal lane work, the runner assembles manifest provenance, normalized
artifacts, rejections, warnings, and an optional accepted chunk map. When an
artifacts, rejections, final grouped diagnostics, and an optional accepted chunk map. When an
output policy selected evidence lanes, it decodes accepted serialized normalize
outputs through their registered codecs and invokes the prepared typed
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
@@ -142,6 +188,15 @@ The CLI publishes those files only after the runner returns without a framework
error. Logical file names and schemas are defined by the [output integration
contracts](../integrations/).
For every completed producer disposition, the runner projects one bounded
validation summary to the manifest, the affected rejection when present, and
the CLI result receipt. The summary records status, configured-order rejecting
validators and reason codes, incomplete validators, producer-attempt count,
and terminal action. It contains no operator message, correction guidance, or
model response. `complete`, `rejected`, and `incomplete` describe the final
candidate disposition; a run-level `incomplete` status indicates at least one
current-run output advanced under `warn_continue`.
## Checkpoint And Debug Hooks
The runner receives checkpoint and debug interfaces rather than roots. It
@@ -151,6 +206,20 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
Selective recomputation can require a canonical accepted normalized predecessor
before a dependent lane starts.
The runner writes successful checkpoint artifacts only after complete accepted
validation. Chunk plans follow the same rule for publication. A rejection,
invalid structured response, or incomplete validation is never reusable state;
the current run may still hand off an otherwise valid `warn_continue` result
according to its terminal policy. The runner carries private reuse eligibility
through extract, merge, normalize, and generated-reference handoff. Any stage
derived from incomplete validation skips both checkpoint lookup and all
checkpoint publication even when that stage's own validation completes.
External references and fully validated generated references remain eligible.
Attempt debug records retain safe kind,
validator, repair-usage, policy, and terminal-decision provenance. Full
assistant and correction content remains confined to the requested detailed
LLM trace.
Debug recording is attempt-scoped and application-owned. A failure to persist
required debug data is a framework error. State roots, persistence, reason-code
meanings, resume, and cleanup are intentionally owned by

View File

@@ -64,12 +64,13 @@ Ordinary resume loads extract, merge, and normalize checkpoints progressively
and may execute later lane stages after an earlier cache miss. Selective
recomputation instead asks the loader for the required producer's accepted
normalize artifact. That lookup reuses the existing normalize files, requires
workspace schema v3 plus an exact non-empty invocation identity, and deliberately
workspace schema v4 plus an exact non-empty invocation identity, and deliberately
does not require extract or merge checkpoint files or dependency fingerprints.
The runner performs canonical codec and producer-provenance validation before
cloning the artifact into normal step output. Success restores only stored
normalize warnings and emits one normalize decision; failure retains the files,
records the decision, and stops without executing the producer or consumer.
normalize diagnostics and emits one normalize decision; failure retains the
files, records the decision, and stops without executing the producer or
consumer.
The loader assigns a typed category and reason code at each validation site;
diagnostic prose is not classified after the fact. The runner then applies
@@ -97,7 +98,7 @@ owns the operator workflow and stable reason-code meanings.
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
resolution, run, warning, and failure artifacts. `internal/framework/debug`
resolution, run, final grouped diagnostic, and failure artifacts. `internal/framework/debug`
implements the pipeline-facing trace recorder under the trace root.
The CLI allocates a bundle before pipeline resolution and treats requested

View File

@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
who need implementation mechanics should read [Run State Internals](internal/state.md).
## Source Deployment
Linux is the supported deployment platform. Install a pinned source release
with the Go version declared in `go.mod` (currently Go 1.25.5):
~~~sh
GOWORK=off go install \
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
~~~
Pin the exact tag in deployment automation rather than following a branch.
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
The maintainer publication process, including tag guards and verification,
belongs to [Source Releases](release.md). macOS builds are best-effort for
development, and Windows is unsupported.
## State Surfaces
Each run can use independent roots with different retention and access-control
@@ -72,6 +89,9 @@ provider call or credentials:
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
~~~
An unset optional `api_key_env` reaches the provider without authorization and
may receive a 401 or 403 response.
Profile paths are currently resolved from the process working directory, not
from the configuration file. The complete example's
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
@@ -85,10 +105,40 @@ and resolves configuration before module preparation and source parsing. It
then performs any permitted cache lookup, executes the pipeline, and publishes
logical output files only after a successful runner result.
On success, the command reports the output bundle path. A warning-bearing run
still succeeds and reports its warning count on standard error. Errors and
On success, the command reports the output bundle path. A run with actionable
process warnings still succeeds and reports warning-group and occurrence counts
on standard error; advisory and observation findings do not produce a warning
line. Errors and
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
## Validation Retries And Terminal Outcomes
Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from
PromptKit's structural-repair calls inside one completion and from an
LLM-backed validator's own retry budget. A semantic correction rebuilds the
ordinary producer request and supplies only the latest rejected model response
plus aggregated validator guidance; it is not a conversation replay.
After the applicable budgets are exhausted, the resolved
[`validation_policy`](config.md#pipelines) determines the result. Structural
failure and semantic rejection normally fail the run; an explicit
`reject_output` records a rejection and allows unrelated work to finish. A
validator execution failure normally uses `warn_continue`, which keeps an
otherwise accepted result in the current run with `incomplete` validation
provenance. It emits one bounded warning for every validator whose execution
budget was exhausted. A corrected result that later passes validation does not
retain diagnostics from abandoned attempts.
Treat a successful process exit as a completed run, not as proof that every
candidate was fully validated. Inspect the receipt's `validation_status`,
`validation_summaries`, rejection count, and warning group and occurrence
counts when an orchestrator requires complete validation. The durable fields
and their meanings are owned by the
[run-result receipt](integrations/run-result.md) and
[published JSON output contract](integrations/json-output.md).
## Output Bundles
Each successful run receives a generated safe run identifier and writes beneath:
@@ -110,9 +160,9 @@ are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
[evidence context](integrations/evidence-context.md) contains source-unit text
and metadata. It is not a cache or debug artifact: retain it with the output
bundle only for as long as consumers need it, and apply source-content access
controls to the entire bundle. Selected lanes may collectively cite most of a
transcript, so a broad allowlist can make the evidence artifact nearly as
sensitive and large as the source itself.
controls to the entire bundle. Its selected source-unit excerpt may include
every source unit once when coverage is broad or its configured window is
large, so do not assume a byte or token reduction or reduced sensitivity.
## Chunk-Plan Cache
@@ -138,7 +188,9 @@ The configured cache mode controls one invocation:
A reused plan is still materialized and validated against the current source.
If a prior plan no longer gives acceptable results, use a refresh run rather
than editing cache files. Deleting a plan is recoverable but can repeat costly
chunking work.
chunking work. A plan accepted only under incomplete validation is not
published, and a rejected cache hit falls through to ordinary generation rather
than becoming a correction candidate.
## Checkpoint Recording, Resume, And Recompute
@@ -163,6 +215,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
runtime identities, so checkpoints created under one state are not reused by
either of the others.
Only accepted, completely validated chunk, extract, merge, and normalize
results are checkpointed for reuse. Rejected, structurally invalid, and
validation-incomplete producer results remain non-reusable, even when a
`warn_continue` result advanced during its original run. A resumed invocation
therefore reruns that producer rather than treating degraded state as accepted.
Checkpoint state is confined below an identity-specific path:
~~~
@@ -217,13 +275,16 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
~~~
The summary contains redacted invocation and resolution information plus run,
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
contains allowlisted application diagnostic records and can include source or
derived application data. Neither surface is a cache input. Do not treat a
debug bundle as safe to share merely because its configuration summary is
redacted. Invocation metadata omits reasoning effort when it is inherited,
records the replacement value when one is supplied, and records an empty value
when inherited reasoning was explicitly cleared.
final grouped diagnostic, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
terminal records contain bounded attempt kinds, validator outcomes, policy,
decision, PromptKit repair count, and usage; they do not contain assistant
responses or complete correction messages. The trace contains allowlisted
application diagnostic records and can include source, model, and correction
content. Neither surface is a cache input. Do not treat a debug bundle as safe
to share merely because its configuration summary is redacted. Invocation
metadata omits reasoning effort when it is inherited, records the replacement
value when one is supplied, and records an empty value when inherited reasoning
was explicitly cleared.
Notarius never creates debug state without an explicit request and never
automatically deletes a requested bundle. If allocation succeeds, the command
@@ -255,14 +316,29 @@ Provider execution settings and the generation timeout come from the selected
PromptKit profile. The invocation-only **--reasoning-effort** and
**--clear-reasoning-effort** controls may replace or clear that profile setting
for all LLM-backed calls in one run without changing the profile. PromptKit
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
complete module operation and validation chain as defined by
[module bindings](config.md#module-bindings-and-validators).
structural output repair happens within one structured-completion call. Its
effective `structured_output_repair_attempts` limit is resolved from the
selected binding, then the pipeline, then the prompt declaration; see
[module bindings](config.md#module-bindings-and-validators). This is distinct
from Notarius binding **retries**, which rerun the complete module operation
and validation chain and do not consume or replenish the structural-repair
limit. The maintained production prompts declare one repair attempt, paid only
after a structural failure. One structured completion with repair budget **R**
makes at most **R + 1** serial provider calls. If one stage attempt performs
**C** structured completions, a binding with **retries: N** has a maximum of
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
corresponding invocation counts and budgets. This is an upper bound, not a
promise that every call reaches a provider.
Timeouts are layered. Caller cancellation is the outer authority. A positive
effective generation timeout adds an inner request deadline, while zero
disables only that generation deadline. The HTTP client timeout remains a
transport-wide cap. Notarius does not add another timeout around PromptKit.
Repairs are serial within the same caller context, so their worst-case latency
and cost follow the provider-call bound above; provision run deadlines and
provider budgets accordingly. Credentials remain optional unless the selected
PromptKit profile requires one, in which case preparation fails before a
provider call when its configured credential is unavailable.
The pinned upstream boundary and profile-format links are in
[PromptKit Integration](integrations/pkg-promptkit.md).
@@ -279,6 +355,11 @@ positive value makes the effective active local-generation bound the smaller
of **total_llm** and that local limit, so a local limit of four permits no more
than four active local generations.
The Notarius scheduler admits one logical structured completion and holds that
permit while PromptKit performs its serial corrective calls. PromptKit applies
its selected-backend admission to each provider call; Notarius does not
reacquire a permit or add another scheduler for a repair.
For a positive local limit, PromptKit owns its default waiting capacity and
admission behavior. When a PromptKit backend has admitted all active and queued
work, a new call fails as capacity exhaustion before generation. The adapter

View File

@@ -24,6 +24,12 @@ DAGs or a general workflow language. Every stage remains explicit; general
chunking, merging, or normalization behavior must not be hidden inside an
extractor.
A stage module is one configured implementation of one pipeline stage. An
artifact family is the cohesive domain feature that owns an artifact across
the explicit stages and supporting codecs, validators, prompts, identity
rules, and reference projections. Artifact-family ownership does not combine
stages or alter the fixed pipeline.
Input and chunking are pipeline-wide. Each selected artifact lane owns its
extract, merge, and normalize stages, and the output stage aggregates the run's
lane outcomes.
@@ -39,6 +45,12 @@ implementations. Domain-neutral model and framework layers provide reusable
policy, contracts, and orchestration. Concrete input, pipeline, output, and
validation extensions depend inward on those generic layers.
Semantic reconciliation is one such domain-neutral framework mechanism. It
prepares bounded source context, invokes a shared model-judgment protocol,
validates proposals, and applies safe plans through typed policies supplied by
the consuming artifact family. It does not own domain identity, durable IDs,
warning semantics, or artifact construction rules.
Generic layers must not depend on production extensions. Concrete extensions
must not compose the application or take ownership of process behavior. The
current packages implementing these layers are inventoried in
@@ -134,8 +146,9 @@ lanes, validators, and LLM profile: the canonical source digest selects the
plan, while the current run still applies its configured chunk validators to
the materialized chunks.
The framework owns orchestration and handoff provenance. Modules return logical
results and warnings; they do not own CLI reporting, physical output, cache, or
The framework owns orchestration, origin enrichment, aggregation, and handoff
provenance. Modules return logical results and classified diagnostics; they do
not own CLI reporting, physical output, cache, or
debug roots, durable file placement, or checkpoint and debug lifecycle.
After pipeline-wide chunking, extraction uses bounded framework concurrency.
@@ -147,26 +160,46 @@ may overlap. The framework must not create unbounded goroutines per lane or
chunk.
Completion timing does not choose public ordering or errors. The coordinator
orders accepted artifacts, warnings, rejections, checkpoint events, and
orders accepted artifacts, grouped diagnostics, rejections, checkpoint events, and
framework errors by stable pipeline scope. Rejections do not cancel unrelated
work. A framework error cancels derived work, prevents undispatched work from
starting, waits for started work, and prevents output encoding.
Warnings are process-only signals: configuration degradation, approved fallback,
or incomplete configured validation. Quality uncertainty and grounding findings
are advisories; successful canonicalization and cleanup are observations.
Modules choose that semantic classification, while the framework attaches
origin, aggregates groups, enforces bounds, and presents final collections.
An ordinary successful run therefore has zero warnings. See
[ADR-0015](../adr/0015-separate-process-warnings-from-quality-diagnostics.md)
for the decision rationale.
## Validation
Validation is a framework-managed boundary around outputs from chunk, extract,
merge, and normalize stages. Validators receive immutable stage output
and make an explicit whole-output decision: approve, approve with warnings, or
reject.
merge, and normalize stages. Validators receive immutable stage output and
make an explicit whole-output decision: approve, reject, fail, or skip when a
runtime prerequisite is unavailable.
Typed artifact validators receive the domain value directly. Chunk validators
receive source-zone chunks, while serialized validators receive immutable
representation bytes and declared schema metadata. A validator registered for
one target or artifact kind cannot satisfy an incompatible selection.
Rejection is a recorded pipeline outcome, not a framework execution error.
Validator execution failures are framework errors. Rejected output does not
advance to the next stage.
The framework runs every applicable validator sequentially in configured order.
It aggregates rejections, exhausted validator failures, and skips before the
producer policy chooses a disposition. A completed rejection never advances.
With no rejection, an exhausted validator failure may fail the run or, under
the configured `warn_continue` policy, advance a structurally valid candidate
with explicit incomplete-validation provenance. Validators report findings;
they do not choose candidate disposition.
A completed rejection supplies a stable reason code for internal provenance
and bounded actionable correction guidance for the candidate producer. Reason
codes, validator keys, and operator-facing messages remain diagnostic data;
they are not model instructions. The framework constructs model-facing retry
text only from the semantic guidance and fails the contract rather than
inventing or truncating missing guidance.
Default validator chains are production composition policy and are registered
centrally by stage and module. Configuration may replace a stage-local default,
@@ -183,11 +216,29 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
and interpretation of structured output. Provider adapters do not own source-
or domain-specific prompt logic.
PromptKit owns bounded structural correction within one structured completion.
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
the two budgets must remain separate.
An LLM-backed producer can participate in semantic correction only when it
declares `single_response_v1` and returns the exact one response that directly
controlled its candidate. On an actionable rejection, the framework rebuilds
the ordinary request and appends only the latest defective response as an
`assistant` message plus one aggregated `user` correction message. This is a
fresh replacement request, not a growing conversation. The retry budgets,
terminal policy, and sensitive-data rationale are recorded in
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md).
When a model selects an application entity, callers must supply a contextual
selection and deterministically attach the opaque application identity whenever
the selection resolves exactly. Models do not receive or reproduce opaque
application identifiers; [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
records the rationale and limited request-local-label exception.
application identifiers. Semantic reconciliation may instead expose
contiguous, one-based candidate handles that exist only for one request;
deterministic code resolves them before typed application, and they never
become durable identity. This is the approved request-local-label application
of [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
recorded by
[ADR-0013](../adr/0013-use-request-local-candidate-handles-for-semantic-reconciliation.md).
LLM calls and other external operations accept cancellation and respect
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
@@ -212,7 +263,8 @@ invalid or incompatible.
Run manifests record enough resolved pipeline, module, source, reference, and
LLM provenance to make a run auditable after configuration changes. Manifests
record identities and summaries rather than secret or large payload content.
record identities and bounded validation summaries rather than secret, raw
model, correction, or large payload content.
## State, Output, And Safety
@@ -232,18 +284,39 @@ an invocation that explicitly requests resume. Debug is never a cache input and
is never created without an explicit request. Pipeline modules receive
collaborator interfaces and never physical roots.
Only accepted, completely validated producer output is reusable checkpoint or
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
results cannot become cache or checkpoint inputs, even when a
`warn_continue` result is allowed to advance in the current run. This
ineligibility follows derived merge and normalize results and generated
references for the remainder of the run: current-run handoff remains allowed,
but no dependent cache or checkpoint may be loaded or published.
Writes are atomic where practical. Paths for writes, moves, overwrites, and
deletion must be narrow and explicit. Notarius never automatically deletes
output or requested debug bundles; cache cleanup is explicit and recoverable.
Secrets must not appear in errors, logs, output, cache, debug summaries,
traces, manifests, documentation, examples, or redacted configuration. Debug
traces, manifests, documentation, examples, or redacted configuration. Raw
assistant responses and complete correction messages are attempt-local and are
excluded from ordinary durable records and summaries; the requested detailed
debug trace is the sole diagnostic surface allowed to retain them. Debug
collection is allowlisted to application-owned payloads and must not capture
unrelated process environment values or filesystem content. Trace data may
contain application data and therefore inherits its sensitivity; operators own
access controls and retention. Physical layout and operation are defined in
[Operations](../operations.md).
## Platform And Distribution
Linux is the supported deployment platform. macOS is supported only as a
best-effort development and compilation environment, while Windows is
unsupported. Notarius distributes source releases only: an immutable source
tag and its checked-in release note identify a release. The project does not
publish executable binaries, archives, installers, container images,
checksums, signatures, or package-manager entries. Maintainer release commands
and tag guards belong to [Source Releases](../release.md).
## Architectural Non-Goals
Notarius does not aim to provide:

View File

@@ -65,6 +65,8 @@ secret values.
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
crosses these topics, choose the document that owns the task and link to the
other contracts.
### Releases
`docs/release.md` owns the source-release procedure. Release notes are
historical summaries, not current-state contract owners: the checked-in note at
an immutable tag records that release, while current canonical documentation
must change with the behavior it describes. Do not use a release note to defer
or replace current documentation updates.
### Contracts And Implementation
Integration and API documents define externally observable shapes and

164
docs/release.md Normal file
View File

@@ -0,0 +1,164 @@
# Source Releases
This procedure is for maintainers publishing Notarius source releases. A
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
candidate after publication; it does not publish or repair a release.
Notarius publishes no binaries, archives, checksums, signatures, containers,
package-manager entries, or Gitea release objects. Windows is not supported.
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
`v0.3.0` tags.
## Select And Describe The Release
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
documented CLI, configuration, durable artifact, integration, or operating
contract when its note explains the impact and required operator action. A
patch release must not intentionally break those documented contracts within
its minor line.
Create the version-matched note as part of the candidate. Every new note uses
this structure, with concise, truthful content in each section:
```markdown
# Notarius vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
The note is a historical summary. Link to current canonical documentation for
exact behavior, and update that documentation in the candidate rather than
using the note as a substitute.
## Prepare The Candidate
Set the selected release version and disable Go workspace use for every
candidate command:
```sh
RELEASE_VERSION=vMAJOR.MINOR.PATCH
export RELEASE_VERSION GOWORK=off
```
Run the shared source-candidate checks from the repository. They cover module
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
configuration validation, and the Linux and Darwin command-build matrix:
```sh
./scripts/check-release-source.sh "$RELEASE_VERSION"
```
Before committing, manually follow every changed local Markdown link and
review the candidate for unintended files, generated output, credentials, or
other unrelated changes. Commit the release note and all affected current
documentation, then run the shared checker against that exact candidate. Push
the candidate commit to `main` only after it succeeds. Record the exact commit
only after that push:
```sh
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
export RELEASE_COMMIT
```
For private-module installation, configure standard `GOPRIVATE` matching this
module and ordinary Git authentication for the hosting service before running
the verification below. The exact authentication mechanism belongs to the
maintainer environment; never record credentials or environment dumps in a
release note, command history, or repository file.
## Guard And Publish The Tag
Fetch current remote references, then run this guard without editing the
candidate. It requires `main`, a clean worktree and index, disabled workspace
use, a stable release version, the recorded and pushed commit, a matching note,
and unused local and remote tags:
```sh
git fetch origin main --tags
if ! printf '%s\n' "$RELEASE_VERSION" |
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
then
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
exit 1
fi
test "$GOWORK" = off
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
test -s "docs/releases/$RELEASE_VERSION.md"
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
done
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
```
Create an explicitly lightweight tag against the guarded commit, verify its
target, and push only that tag ref:
```sh
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
```
Never use `git push --tags`, move a published tag, or delete a published tag.
## Verify The Published Release
Confirm that the remote tag still points at the guarded commit and that the
note is available from the tagged tree:
```sh
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
```
Verify a fresh source installation and its diagnostic version. The temporary
directory confines the installed command to this check:
```sh
release_verification_dir=$(mktemp -d)
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
mkdir -p "$release_verification_dir/bin"
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
```
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
equivalent source verification when local installation policy requires it.
`notarius --version` is diagnostic only; downstream compatibility remains
defined by the published receipt and artifact contracts.
## Failure And Correction Policy
If candidate validation fails before publication, fix the candidate on `main`,
rerun the shared checker, and repeat the guards. An unpublished local tag may
be deleted after inspection.
If the remote tag or tag CI reveals a defect, leave the published tag intact.
Fix the defect on `main`, choose a new patch version, write a new matching
note, and repeat this procedure. Do not weaken tag immutability or add release
assets as a workaround.

83
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,83 @@
# Notarius v0.4.0
This release strengthens LLM reliability and validation throughout the
configured pipeline, upgrades the PromptKit integration, and establishes the
source-release and downstream-consumer workflows needed for broader D&D
pipeline integration.
## Summary
Notarius now distinguishes PromptKit structural-output repair from
application-owned semantic validation retries. Producer candidates can run
through complete deterministic validator chains, receive bounded semantic
correction guidance, and retry under explicit stage policies. Final run
receipts and manifests preserve bounded validation provenance, while outputs
that advance with incomplete validation remain available to the current run
without entering reusable checkpoint state.
The release also adds a maintained complete D&D subprocess-consumer workflow,
diagnostic build versions, and the source-only release procedure used to
publish this version.
## Compatibility
- Configuration files must use schema version 4. Version 3 is not decoded or
rewritten; rename the top-level `scriptorium` section to `promptkit` when
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
format and may use its profile-inheritance support. Notarius continues to
resolve operator profiles before embedded fallbacks.
- Structural-output repair and semantic stage retries are separate bounded
mechanisms. Maintained production prompts request one structural repair by
default; explicit configuration can override the supported repair count.
- Validation policy can now fail a run, reject an output, or permit an
otherwise valid candidate to advance with incomplete-validation provenance.
The application defaults are documented in
[Configuration](../config.md#pipelines).
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
optional validation summaries plus a required validation-status field.
Consumers of this pre-release contract should follow the current
[run-result receipt](../integrations/run-result.md).
- Existing D&D artifact schema identities remain unchanged. Validation and
producer-policy changes can nevertheless cause previously accepted weak
candidates to retry, reject, or fail instead.
## Upgrade
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
to `promptkit`.
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
and ensure their credential environment variables are available at run
time.
3. Run `notarius config validate --config <path> --pipeline <id>` before the
first production invocation.
4. Review `structured_output_repair_attempts`, producer retry counts, and
`validation_policy` wherever the deployment needs behavior different from
the documented defaults.
5. Update subprocess consumers to inspect receipt `validation_status` and to
tolerate the optional bounded `validation_summaries` field. A consumer that
requires fully validated artifacts should require `approved`.
## Changes
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
inheritance, structured-output repair, typed error classification, and the
correction-aware completion protocol.
- Added pipeline and binding configuration for structural repair and terminal
validation policy, with strict startup validation and effective-setting
provenance.
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
semantic reconciliation producers. Retry prompts contain the exact defective
response and actionable semantic correction guidance without exposing
internal reason codes or opaque entity identifiers.
- Added complete validator-chain execution, validator retry handling, bounded
warnings, terminal dispositions, and durable validation summaries.
- Prevented validation-incomplete artifacts and all derived lineage from
loading or publishing reusable checkpoints while preserving same-run
generated-reference handoff.
- Tightened cached chunk-plan validation so only completely validated plans are
reused or replace stored plans.
- Added a machine-readable subprocess receipt workflow and complete D&D
consumer documentation covering all maintained artifacts.
- Added source-release checks, immutable lightweight-tag guidance, Linux and
Darwin build verification, and diagnostic `notarius --version` output.

85
docs/releases/v0.5.0.md Normal file
View File

@@ -0,0 +1,85 @@
# Notarius v0.5.0
This release separates actionable process warnings from extraction-quality
advisories and routine normalization observations, giving operators a quiet
warning channel without discarding durable diagnostic detail.
## Summary
Notarius now carries one validated, origin-aware diagnostic contract from
producers and validators through retries, reusable state, output publication,
debug summaries, run receipts, and CLI presentation. Warnings are reserved for
process degradation or incomplete configured work. Data-quality findings are
advisories, and successful deterministic cleanup is recorded as observations.
An ordinary successful run therefore reports zero warnings while retaining
bounded diagnostic provenance for later review.
The framework aggregates findings deterministically by their stable identity
and complete pipeline origin, preserves exact occurrence counts, and retains
bounded representative samples. Warning groups fail rather than truncate;
advisory and observation representation is bounded with explicit truncation
metadata and exact unrepresented-occurrence counts.
## Compatibility
- `warnings.json` now uses the incompatible grouped
`notarius.warnings.v2` envelope and contains process warnings only. Consumers
of the former flat warning payload must migrate to the current
[JSON output contract](../integrations/json-output.md).
- The new `diagnostics.json` file uses `notarius.diagnostics.v1` and contains
advisory and observation groups. Production `index.json` files always expose
both `warnings_file` and `diagnostics_file`.
- The machine-readable run receipt is now `notarius.run-result.v2`. It replaces
`warning_count` with exact warning group and occurrence counts and adds
advisory/observation group, occurrence, and truncation fields. See the
current [run-result receipt](../integrations/run-result.md).
- Custom output modules must return their complete logical file set or an
error. The former `OutputResult.Warnings` field has been removed; an output
module cannot report a warning after serializing its output.
- Reusable state now uses `notarius.workspace.v4` and chunk-plan records use
`notarius.chunk-plan.v3` so they can preserve structured diagnostics. Older
pre-release reusable state is not reused under these contracts; start with
clean state when deterministic continuity with an older workspace is not
required.
- Validation acceptance, semantic retry budgets, rejection policy, and D&D
artifact schema identities are unchanged by this release.
## Upgrade
1. Update subprocess consumers to require `notarius.run-result.v2` and read
`warning_group_count`, `warning_occurrence_count`,
`diagnostic_group_count`, `diagnostic_occurrence_count`, and
`diagnostics_truncated`.
2. Update output-bundle consumers to decode `notarius.warnings.v2`, discover
`diagnostics.json` through `index.json`, and treat diagnostics as review
information rather than process warnings.
3. Update any custom output module for the removal of
`OutputResult.Warnings`; return an error when encoding cannot complete.
4. Clear pre-release reusable state before the first upgraded production run
when deterministic continuity with an older workspace is not required.
5. Run `notarius config validate --config <path> --pipeline <id>` and perform
one representative run before promoting the release in an automated
pipeline.
## Changes
- Added validated diagnostic dispositions, categories, origins, stable reason
codes, exact occurrence counts, and bounded representative samples.
- Added deterministic run-level aggregation with separate limits for
actionable warning groups and advisory/observation groups.
- Reclassified D&D source-relatedness and unresolved-identity findings as
data-quality advisories and routine normalization changes as observations.
- Preserved structured diagnostics across producer retries, validation,
generated-reference handoff, checkpoints, chunk-plan reuse, and debug
summaries while discarding superseded-attempt findings.
- Added grouped `warnings.json`, a new grouped `diagnostics.json`, and the
corresponding production index entries.
- Upgraded the machine-readable run receipt and human CLI summary to report
exact warning and diagnostic counts without allowing advisory volume to
create warning output.
- Removed post-encoding output warnings and hardened diagnostic validation,
overflow handling, aggregate memory bounds, and warning-file path
presentation.
- Documented diagnostic ownership, classification, operator interpretation,
durable contracts, and architectural invariants in ADR-0015 and the
canonical CLI, operations, integration, and internal documentation.

View File

@@ -0,0 +1,648 @@
# Warning Signal And Presentation Audit
## Executive Assessment
Notarius warning execution is mechanically stronger than its operator-facing
presentation. Terminal-attempt promotion, stable ordering after concurrent
work, checkpoint replay, validation summaries, and debug retention are all
substantially correct. The audit found no general duplicate-append defect in
the extract, merge, or normalize handoffs and no leakage of abandoned-attempt
warnings into a successful result.
The warning channel itself is not coherent. One flat `contracts.Warning` type
currently represents at least four materially different concepts:
- actionable degradation or incomplete validation;
- heuristic data-quality doubt;
- successful but potentially reviewable fallback; and
- routine canonicalization, ordering, and deduplication observations.
That conflation is the primary reason successful runs produce a count that is
large but operationally weak. The maintained complete D&D example demonstrates
the problem without a live provider: an approved run with no rejected outputs
published 12 warning records, all from three advisory relatedness checks. An
operator separately reported a successful complete D&D run with 10 outputs,
one rejection, and 85 warnings. The production bundle for that run was not
available in this environment, so its reason-code distribution could not be
measured.
The current implementation also has four correctness or robustness gaps:
1. warning records lose stage, step, lane, module, validator, and chunk
provenance when promoted, which makes safe aggregation and diagnosis
impossible from `warnings.json` alone;
2. there is no framework-level validation or aggregate bound, and the NPC- and
spell-relatedness validators bypass the D&D warning limiter entirely;
3. warnings returned by an output encoder are added after `warnings.json` has
already been encoded, so the receipt, stderr, debug bundle, and published
warning file can disagree; and
4. a skipped validator contributes to `incomplete` validation but does not
receive the warning generated for an exhausted validator failure.
The recommended end state is a structured diagnostic contract with explicit
disposition, category, origin, occurrence count, and bounded samples. Warnings
are reserved for process-level degradation or incompleteness. LLM-judged or
deterministically inferred extraction-quality signals are advisories, never
warnings, and routine normalization observations remain inspectable without
being reported as top-level warnings. An ordinary successful run in which all
configured work completes normally should therefore report zero warnings. This
is an architectural and durable-contract change, not merely revised CLI prose.
## Evidence And Limits
The audit used:
- a complete static search of production `contracts.Warning` constructors,
reason-code constants, result fields, and promotion sites under `internal/`;
- call-path inspection through producer attempts, validators, lane
coordination, chunk-plan reuse, checkpoints, output encoding, debug output,
CLI presentation, and run-result construction;
- the maintained complete and minimal D&D examples with offline fake LLMs;
- focused deterministic tests for warning bounds, semantic-reconciliation
fallback, `warn_continue`, semantic retries, terminal rejection, concurrency
ordering, and checkpoint reuse; and
- the operator-provided observation of an 85-warning complete D&D run.
No provider-backed production run was attempted because this environment has
no API key. Consequently, the audit can establish warning mechanics, possible
multiplicity, synthetic volume, and obvious heuristic limitations, but cannot
estimate production frequency or the real false-positive rate of individual
D&D advisories. Those measurements are not required to choose the recommended
architecture; they are required before strengthening any heuristic advisory
into a rejection or setting a numerical production acceptance target.
## Complete Warning-Producer Inventory
### Framework And Generic Boundaries
| Producer | Reason code | Trigger and consequence | Multiplicity and bound | Current surfaces and coverage |
| --- | --- | --- | --- | --- |
| Reference materialization in `internal/framework/pipeline/references.go` | `empty_reference` | A bound external reference is a valid, accepted media type but contains zero bytes. The prompt may receive materially incomplete context. | One per empty bound file; finite by configuration but no shared run-level cap. | Enters `RunInput.Warnings`; reference tests protect contextual scope. |
| Producer-attempt policy in `internal/framework/pipeline/producer_attempts.go` | `validator_execution_incomplete` | An applicable validator exhausted its execution budget and `warn_continue` accepted the otherwise valid candidate. | One per failed validator on each terminal candidate. An extract chain can multiply this by chunks and lanes. There is no global cap. | Durable warning, receipt count, stderr, debug, and validation summary. `TestWarnContinueRecordsOneWarningForEachExhaustedValidator` covers failures. |
| Chunk, extract, merge, normalize, and output module result contracts | Module-defined | A module may return arbitrary warnings with its successful candidate. | No contract validation, message limit, per-result cap, or global cap. Current production modules are inventoried below. | Terminal-attempt filtering and concurrency ordering are well tested. |
| Production JSON output encoder | None | The encoder copies incoming warnings into `warnings.json`; it does not currently create warnings. | Same count as its input. | JSON encoder and assembled-pipeline tests compare the incoming run warnings with the published file. |
| Output encoder result contract | Module-defined | Any output encoder may return warnings discovered during encoding. | Unbounded by contract. No production encoder currently exercises this capability. | Appended to final `RunOutput` only after logical files were encoded; this is the cross-surface defect described in AUD-WARN-004. |
Input adapters and production mergers do not currently have independent
warning producers. Chunk-plan and checkpoint decisions are structured manifest
or debug provenance rather than warnings. Cancellation and hard persistence,
reference, parsing, serialization, and provider failures remain errors.
### D&D Extraction Gates
| Producer | Reason code | Trigger and consequence | Multiplicity and bound |
| --- | --- | --- | --- |
| `dnd/combat-turns` extractor | `scene_classification_unavailable` | The chunk has no exact matching scene-description classification. The extractor returns an empty accepted result and skips the LLM, so combat-turn output may be incomplete. | At most one per chunk for this lane. |
| `dnd/enemy-events` extractor | `scene_classification_unavailable` | The same missing or mismatched scene gate causes accepted empty enemy-event output. | At most one per chunk for this lane. |
An exact non-combat classification produces an intentional empty result without
a warning. An exact combat classification proceeds normally. The two producers
share a code and operator consequence but use different messages; their module
origins are not retained in the final warning record.
### D&D Source-Relatedness Validators
All ten relatedness validators are deterministic advisories: they approve the
candidate and warn when contextual prose or an entity name is not lexically
present in cited text. Shape and source-reference failures are deliberately
left to blocking validators earlier in the chain. The same relatedness
validator is registered in both the extract and normalize default chain for
each artifact family in `internal/modules/dnd/register/chains.go`.
| Artifact family | Warning reason | Per-record trigger | Local bound | Omission reason |
| --- | --- | --- | --- | --- |
| Combat turns | `combat_turn_not_near_source` | Actor token sequence absent | 20 per validator invocation | `combat_turn_relatedness_warnings_omitted` |
| Enemy events | `enemy_event_not_near_source` | Subject token sequence absent | 20 | `enemy_event_relatedness_warnings_omitted` |
| Item occurrences | `item_occurrence_source_unrelated` | Item name token sequence absent | 20 | `item_occurrence_relatedness_warnings_omitted` |
| Item registry | `item_not_near_source` | Item name token sequence absent | 20 | `item_relatedness_warnings_omitted` |
| Location occurrences | `location_occurrence_not_near_source` | Location name token sequence absent | 20 | `location_occurrence_relatedness_warnings_omitted` |
| Location registry | `location_not_near_source` | Location name token sequence absent | 20 | `location_relatedness_warnings_omitted` |
| NPC occurrences | `npc_occurrence_not_near_source` | NPC name token sequence absent | 20 | `npc_occurrence_relatedness_warnings_omitted` |
| NPC registry | `npc_not_near_source` | NPC name token sequence absent | **Unbounded** | None |
| Scene descriptions | `scene_description_not_near_source` | No significant title or summary token appears; up to two findings per scene | 20 | `scene_description_relatedness_warnings_omitted` |
| Spells | `spell_not_near_source` | Spell-name token sequence absent | **Unbounded** | None |
The eight limiter-generated omission records are presentation artifacts, not
new source-relatedness conditions. They occupy a warning slot and make list
length differ from the actual occurrence count.
### D&D Normalizers
Every production D&D normalizer bounds its returned warning slice to 20 through
`internal/modules/dnd/shared/diagnostics`, including a final omission record
when needed. Registry semantic retries reserve space for their fallback
warning. The following table is complete by semantically distinct condition;
codes listed together are parallel artifact-family variants.
| Condition | Reason codes | Result impact | Current classification assessment |
| --- | --- | --- | --- |
| Display or field whitespace/name canonicalization | `npc_fields_normalized`, `item_fields_normalized`, `location_fields_normalized`, `spell_name_canonicalized`, `combat_actor_canonicalized`, `enemy_event_name_canonicalized`, `item_occurrence_name_canonicalized`, `location_occurrence_name_canonicalized`, `scene_description_prose_normalized` | Deterministic successful mutation. The item-occurrence code can also describe `from`/`to` whitespace, not only the item name. | Routine observation. |
| Durable ID recomputation | `npc_id_recomputed`, `item_id_recomputed`, `location_id_recomputed` | Restores the deterministic name-derived ID. | Routine observation; invalid identity is separately rejected by default chains. |
| Source-reference sorting or deduplication | `source_references_normalized` | Sorts and removes exact duplicate references while deliberately preserving invalid references for their validators. | Routine observation. Shared code is useful but ambiguous without producer origin. |
| Canonical record ordering | `combat_turns_reordered`, `enemy_events_reordered`, `item_occurrences_reordered`, `location_occurrences_reordered`, `npc_occurrences_reordered`, `scene_description_order_normalized` | Deterministic order changes only. | Routine observation. |
| Exact or approved semantic duplicate consolidation | `duplicate_npc_collapsed`, `duplicate_item_collapsed`, `duplicate_location_collapsed`, `duplicate_spell_cast_collapsed`, `duplicate_combat_turn_collapsed`, `duplicate_enemy_event_collapsed`, `duplicate_item_occurrence_collapsed`, `duplicate_location_occurrence_collapsed`, `duplicate_npc_occurrence_collapsed`, `scene_description_duplicate_collapsed` | Removes duplicate records and preserves or combines canonical evidence according to the artifact policy. Registry codes cover both exact and accepted semantic consolidation. | Durable normalization observation; not normally operator-actionable. |
| Unresolved external membership | `spell_name_unresolved`, `item_occurrence_unknown_item_id`, `location_occurrence_unknown_location_id` | The value is preserved but is not grounded in the effective catalog or registry. Default chains normally reject the same condition before normalization; it remains reachable with validator overrides or defensive direct use. | Actionable data-quality warning. |
| Unsafe currency consolidation proposal | `item_semantic_proposal_invalid` | The proposed group is rejected and all records are preserved because denominations or currency/non-currency members are incompatible. The same code is also used internally as a retry reason. | Advisory about model proposal quality; no accepted-data loss. The control and diagnostic meanings should be separated. |
| Semantic reconciliation unavailable or exhausted | `npc_semantic_reconciliation_exhausted`, `item_semantic_reconciliation_exhausted`, `location_semantic_reconciliation_exhausted` | The safe deterministic result is accepted, but possible semantic duplicates remain. | Actionable fallback warning. |
| Local warning truncation | `npc_normalization_warnings_omitted`, `item_normalization_warnings_omitted`, `location_normalization_warnings_omitted`, `spell_normalization_warnings_omitted`, `combat_turn_normalization_warnings_omitted`, `enemy_event_normalization_warnings_omitted`, `item_occurrence_normalization_warnings_omitted`, `location_occurrence_normalization_warnings_omitted`, `npc_occurrence_normalization_warnings_omitted`, `scene_description_normalization_warnings_omitted` | Reports that individual records were omitted from presentation. | Group metadata, not an independent warning. |
No production D&D normalization warning exposes raw model responses,
correction guidance, or provider errors. Most dynamic names are quoted and
truncated by the shared helper. That local discipline is not enforced by the
generic warning contract, and the spell relatedness message does not use the
shared truncation helper.
## Warning Propagation And Surface Map
```text
external-reference warnings -----------------------------+
|
module candidate warnings -> validation chain warnings |
| | |
+---- producer-attempt terminal policy -------+
| |
accepted / terminal rejection only |
| |
chunk or lane result in canonical order |
| |
checkpoint record/replay and ordered step merge |
| |
RunOutput.Warnings <------------+
|
OutputRequest -> output encoder
| |
warnings.json OutputResult.Warnings
|
appended to final RunOutput only
|
receipt, stderr, final debug warning summary
```
### Attempts And Validation
- `runProducerAttempts` promotes only the terminal accepted or terminal
rejected candidate's module and completed-validator warnings. Operational,
structural, semantic, and module-directed attempts that are superseded are
retained in attempt debug artifacts but not in the final collection.
- A module-directed semantic-reconciliation retry adds its fallback warning
only when no retry remains. Earlier attempt warnings are discarded.
- On `warn_continue`, warnings from the otherwise accepted candidate and
completed approved or rejected validators are retained. One fixed,
non-sensitive `validator_execution_incomplete` warning is added for every
failed validator. Skipped validators affect the validation summary and final
`incomplete` status but do not receive such a warning.
- A semantic terminal rejection retains only warnings from that rejected
attempt. Structural rejection after producer failure cannot retain a
candidate warning because no valid candidate result exists.
These behaviors are protected by the producer-attempt, extract-handoff,
rejection-warning, normalize-retry, and attempt-debug tests. They are the right
foundation for the redesign and should not be replaced with early-exit or
all-attempt accumulation.
### Concurrency And Ordering
Extract jobs are dispatched chunk-first and lane-second. Results are stored by
chunk index, finalized in ascending chunk order, and lane continuations are
merged into a slice indexed by configured lane order. Pipeline steps run in
configured order. The resulting public order is therefore:
1. pre-run reference warnings;
2. chunk-stage warnings;
3. step order;
4. configured lane order within each step;
5. chunk order within extract;
6. merge warnings; then
7. normalize warnings; followed by any output-result warnings.
`TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion` exercises warning
order under reversed completion. No completion-order leak was found.
### Chunk Plans And Checkpoints
- A reusable chunk plan stores producer warnings only. Current validators run
again, and their current warnings are appended. Warnings from a cached plan
candidate that fails current validation are discarded before regeneration.
- Accepted extract, merge, and normalize checkpoints store the terminal
warnings for their stage. Reuse loads and appends those warnings once at the
same logical handoff. Tests compare fresh and resumed warning collections and
preserve their order.
- Validation-incomplete accepted outputs are not reusable, preventing a later
run from silently treating incomplete validation as complete.
- Required accepted-normalize hydration replays that normalize checkpoint's
warnings; checkpoint decisions separately expose that reuse occurred.
The recommended redesign should keep fresh and resumed logical diagnostics
equivalent. Whether a result was reused belongs in checkpoint provenance, not
in the diagnostic grouping key; adding a `reused` distinction would fragment
groups and make equivalent runs present differently.
### Terminal Surfaces
| Surface | Current content | Audience | Audit result |
| --- | --- | --- | --- |
| `RunOutput.Warnings` | Flat final slice | Framework and CLI | Canonical in-memory list, but lacks origin and bounds. |
| Published `warnings.json` | Object containing the warnings passed into the output encoder | Durable consumers | Exact for the production JSON encoder unless the encoder itself returns warnings. |
| `index.json` | Path to `warnings.json` | Durable consumers | Stable discovery path; no separate diagnostic-detail path. |
| Run-result v1 | `warning_count = len(final RunOutput.Warnings)` | Subprocess callers | Count only; no group/occurrence distinction. |
| Human stderr | `run completed with N warning(s)` | Operators | Count only and no direct detail path. Successful exit remains zero. |
| Manifest | Validation and rejection summaries, no warning collection | Durable provenance | Correctly avoids duplicating the flat list. |
| Debug summary `warnings.json` | Raw final warning array | Operators/developers | Includes final output-result warnings and can therefore differ from published `warnings.json`. |
| Debug run report | Final warning count | Operators/developers | Same final slice length as receipt and stderr. |
| Attempt/stage debug | Candidate-local warning detail and origin in path/envelope | Forensics | Sufficient to diagnose provenance, but debug capture is optional and is not a durable consumer contract. |
## Empirical Measurements
### Offline And Synthetic Runs
| Scenario | Result | What it establishes |
| --- | --- | --- |
| Maintained complete D&D config and transcript with the repository's offline fake LLM | Approved, 10 normalized outputs, 0 rejected outputs, 12 warnings; receipt, stderr, and published file all reported 12 | An ordinary structurally successful workflow can be noisy without fallback or incomplete validation. |
| Same complete run, grouped after publication | Three reason codes, seven exact `(reason, scope, message)` tuples, maximum exact-tuple repetition of three | The flat count materially overstates distinct operator conditions. Scope resets within chunks and does not identify origin. |
| Maintained focused scene-description workflow | Approved, one normalized output, 0 warnings | The warning channel can be quiet when synthetic model text is lexically grounded. |
| Generic warning publication contract | One warning reaches successful stderr, durable output, and debug summary | The ordinary pre-output path is consistent. |
| NPC semantic-reconciliation candidate-limit fallback | No LLM call, all records preserved, one exhaustion warning, total warnings no greater than 20 | Fallback is bounded and materially different from routine normalization. |
| `warn_continue` with two failed validators and one skipped validator | Validation status contains all three incomplete validators; warning slice contains two execution-incomplete records | Current warning count does not describe all incomplete validation. |
| Retrying extract candidate | Two producer attempts; only the accepted attempt's one warning is final | Retry does not amplify abandoned warnings. |
| Terminal semantic rejection | Only the final rejected attempt's operation and validator warnings are final | Rejection diagnostics are retained without retaining superseded warnings. |
| Fresh versus reused extract checkpoint | Warning collections are deeply equal | Checkpoint replay does not itself amplify warnings. |
| Spell normalizer with 21 unresolved entries | 20 records: 19 samples plus one omission record saying two additional warnings were omitted | `warning_count` is neither exact occurrence count nor distinct-condition count. |
The focused audit tests passed in `internal/cli`,
`internal/framework/pipeline`, the NPC-registry and spell normalizers, and all
D&D packages.
### Bounded Sample Review
The complete offline D&D run produced:
| Reason | Count | Sample | Review |
| --- | ---: | --- | --- |
| `location_not_near_source` | 4 | `Moon Gate` was absent from cited text | Correctly identifies deliberately unsupported fake output. Three records shared the same exact tuple because chunk and stage origin were lost. |
| `location_occurrence_not_near_source` | 4 | A `Moon Gate` visit was absent from cited text | Correctly identifies the same unsupported registry-driven occurrence, but repeats the same operator concern across extraction and normalization. |
| `scene_description_not_near_source` | 4 | A title or summary had no significant exact token in cited text | Mixed value. Generic `session scene` prose is ungrounded, while `Arrival` versus transcript `arrive` illustrates an expected lexical false positive. |
This fake workflow is an integration fixture, not a model-quality benchmark.
It nonetheless proves that the checks carry useful evidence while being too
imprecise and repetitive to serve as one-warning-per-record operator alerts.
### Production Evidence Still Needed
The reported 85-warning run establishes that high volume occurs in practice,
but the following remain unknown:
- dominant production reason codes and stage/lane sources;
- unique group count versus repeated occurrence count;
- false-positive rate for each relatedness family;
- how much volume comes from normalization observations versus advisories;
- whether fresh and resumed production runs remain equivalent; and
- a defensible numerical acceptance target.
If further data is worthwhile, the operator can supply the v1 receipt,
`warnings.json`, and manifest validation summaries without supplying transcript
or lane artifacts. An initial privacy-preserving report should group by reason
code and normalized scope family, count exact repeated tuples, and omit message
text. Reviewing heuristic precision requires a separately approved bounded
sample with its cited source context.
## Classification Of Current Conditions
| Target disposition | Current families | Result impact | Operator action | Durable placement |
| --- | --- | --- | --- | --- |
| **Warning** | Empty reference; validator failure or skip accepted under `warn_continue`; unavailable required scene classification; exhausted semantic reconciliation | A configured process completed under an allowed degraded or incomplete policy rather than completing normally | Correct reference/configuration, inspect provider/validator, or rerun | Actionable `warnings.json`, receipt summary, stderr summary, debug |
| **Advisory** | Source-relatedness heuristics; unresolved spell or registry membership; guarded invalid semantic proposal; any future LLM-judged uncertainty or extraction-quality signal | Uncertain data quality or poor model proposal, but accepted data is structurally valid and deterministic guards prevented unsafe mutation | Optional model/source review; no routine action for every record | Durable diagnostic detail and debug; never a top-level warning |
| **Observation** | Whitespace/name/ID/source-reference canonicalization; canonical ordering; exact and approved semantic duplicate consolidation | Successful intended normalization | None under normal operation | Durable bounded normalization diagnostics or debug; no stderr warning |
| **Not a diagnostic** | Rejection, invalid structure, cancellation, persistence error, provider failure under fail-run policy | Candidate or run did not complete according to policy | Inspect rejection/error and retry or correct input/configuration | Existing rejection, validation summary, error, and debug contracts |
Exact and semantic duplicate consolidation should remain distinguishable in
category or reason metadata even though both are observations. Semantic
reconciliation exhaustion remains a warning because a capability was not
applied; successful approved consolidation is an observation because it is the
normalizer's intended work.
## Ranked Findings
### AUD-WARN-001 — The Flat Warning Type Destroys Signal Quality
- **Priority:** High operator impact; high implementation leverage.
- **Evidence:** `contracts.Warning` has only scope, reason, and message. Routine
normalizer changes, heuristic doubt, fallback, and incomplete validation all
enter the same slice and the same CLI count. The offline complete run's 12
records were all advisories; the operator observed 85 records in a successful
real run.
- **Impact:** Operators cannot tell whether a warning requires a rerun, a
configuration repair, optional review, or no action. Repeated routine output
trains them to ignore the channel.
- **Recommendation:** Replace the flat semantic contract with explicit
`warning`, `advisory`, and `observation` dispositions plus a small category
vocabulary. Do not infer disposition from message text or require every
downstream consumer to maintain a reason-code policy table.
### AUD-WARN-002 — Warning Records Lose The Origin Needed For Diagnosis And Aggregation
- **Priority:** High correctness and usability impact.
- **Evidence:** The runner knows stage, step, lane, module, validator, chunk ID,
and chunk index at promotion time, but `terminalWarnings` flattens module and
validator records into `[]contracts.Warning`. Per-chunk scopes such as
`locations[0]` and `occurrences[0]` then repeat without identifying their
chunk or producer. `source_references_normalized` is intentionally shared
across families and is therefore especially ambiguous.
- **Impact:** `warnings.json` cannot answer which stage or module produced a
record. Message- or scope-based deduplication would merge unrelated findings
or retain accidental duplicates.
- **Recommendation:** Keep module findings free of framework context, then have
the framework add a structured origin envelope before promotion. Validator
findings must retain validator identity instead of passing through
`validationReport.Warnings()` as a flat slice.
### AUD-WARN-003 — Warning Volume Is Not End-To-End Bounded Or Validated
- **Priority:** High robustness impact; medium immediate likelihood.
- **Evidence:** D&D's `LimitWarnings` caps most individual producers at 20, but
NPC- and spell-relatedness return one warning per record without the helper.
Every extract validator is invoked per chunk, all ten relatedness checks run
again after normalization, and there is no run-level collector. The generic
contract validates neither disposition nor reason/message size, UTF-8,
blankness, or total records.
- **Impact:** Warning memory and output grow with chunks, lanes, configured
validators, and record counts. Local omission records lose exact occurrence
semantics while still incrementing `warning_count`.
- **Recommendation:** Add a generic bounded diagnostic collector that preserves
exact occurrence counts and bounded samples. Validate all diagnostic fields
at the module/framework boundary. Immediately bring NPC and spell
relatedness under the existing cap if the full redesign is staged.
### AUD-WARN-004 — Output Encoder Warnings Make Durable Surfaces Disagree
- **Priority:** Medium current impact; high contract correctness risk.
- **Evidence:** `Runner.Run` passes existing warnings to `encoder.Encode`, then
the production encoder serializes `warnings.json`. Only after encoding does
the runner append `OutputResult.Warnings`. The receipt, stderr, debug summary,
and debug run report see the final slice; the already-created published file
cannot. No production encoder currently returns a warning, so ordinary JSON
runs do not trigger the defect.
- **Impact:** A valid output-module implementation can violate the documented
claim that `warning_count` describes the published warning collection.
- **Recommendation:** Remove successful output warnings from the output-module
contract unless a demonstrated use case requires them; encoding failures
should be errors and optional encoder observations should be debug data. A
two-phase finalize API is the viable but more complex alternative.
### AUD-WARN-005 — Validation Skips Are Incomplete But Not Warned
- **Priority:** Medium operator/correctness impact.
- **Evidence:** `firstIncompleteValidation` treats failed and skipped validators
alike, and validation summaries include both. `incompleteValidationWarnings`
emits records only for `validationFailed`. The focused test demonstrates
three incomplete validators but two warnings.
- **Impact:** A successful run can have `validation_status: incomplete` while
its warning count understates or even omits the affected validators. A caller
that checks only warnings receives a weaker signal than the manifest and
receipt status.
- **Recommendation:** Produce one aggregated incomplete-validation warning
group whose occurrences cover both failure and skip, while retaining typed
outcome and safe reason metadata in the validation summary. Do not expose
provider errors or arbitrary skip prose in model or operator messages.
### AUD-WARN-006 — Relatedness Checks Are Useful But Repetitive And Lexically Weak
- **Priority:** Medium operator impact; low acceptance-policy urgency.
- **Evidence:** Every family runs the advisory in both extract and normalize
chains. The complete fixture contains exact repeated tuples, and the checks
rely on exact normalized token sequences or a minimal significant-token
overlap. Reason naming drifts between `*_not_near_source` and
`*_source_unrelated`.
- **Impact:** The checks can catch unsupported entities, but aliases, pronouns,
inflection, and generic scene prose create predictable false positives. Flat
per-record presentation magnifies them.
- **Recommendation:** Retain the validators and their stage-local execution,
but classify and aggregate them as advisories. Normalize reason-code naming
when the diagnostic contract changes. Do not strengthen them into rejection
rules without a human-reviewed production evaluation.
### AUD-WARN-007 — `warning_count` Has No Stable Operational Meaning
- **Priority:** High downstream-contract impact.
- **Evidence:** The receipt and CLI use `len(output.Warnings)`. One list element
can be an omission summary representing several hidden occurrences; repeated
records can represent the same condition; and skipped validators can be
absent. A 21-occurrence spell test produces a list length of 20.
- **Impact:** The value is neither an exact occurrence count nor a distinct
warning-group count. Consumers cannot set policy or present a trustworthy
summary from it.
- **Recommendation:** Introduce explicit warning-group and warning-occurrence
counts in a versioned receipt. Do not silently redefine the v1 field.
## Recommended Target Contract And Presentation Model
### Diagnostic Model
Use one validated internal diagnostic model with these concepts:
- **disposition:** `warning`, `advisory`, or `observation`;
- **category:** a small enum such as `configuration`, `degradation`,
`validation_incomplete`, `data_quality`, `fallback`, or `normalization`;
- **reason code:** stable semantic identity owned by the producer;
- **origin:** framework-added phase/stage, step ID, lane ID, module key,
validator name, chunk ID, and chunk index when applicable;
- **occurrence count:** exact number of matching findings;
- **samples:** a small deterministic list of bounded scope/message pairs; and
- **omitted sample count:** `occurrence_count - len(samples)`, represented as
metadata rather than another diagnostic record.
Errors and rejected outputs must not become diagnostic dispositions. A warning
means that the run completed under policy despite a process-level degradation
or incomplete configured operation. Advisory and observation dispositions can
describe accepted artifact quality and transformation provenance, but no
LLM-judged extraction-quality signal may be promoted to a warning. Validation
status remains authoritative for approval, rejection, and incomplete
validation.
### Aggregation
The framework runner should own aggregation after it enriches findings with
origin and before public output construction. Modules and validators retain
semantic ownership of disposition, category, reason, scope, and message; they
must not own CLI or file presentation.
The default stable key should be:
```text
disposition + category + reason_code
+ phase/stage + step_id + lane_id + module_key + validator_name
```
Chunk, record scope, and message text belong in samples and must not be part of
the group key. This groups repeated per-chunk findings without merging the same
code across distinct producers or pipeline locations. Group order should be
the first occurrence in the runner's existing canonical order; sample order
should follow the same order. A final canonical sort by the complete origin key
is also viable, but completion timing must never choose either order.
Aggregation must be incremental and bounded. Producers should use a shared
collector that counts every occurrence while retaining only bounded samples;
the framework then merges producer groups without reconstructing counts from
omission prose. A global maximum group count is also required, with overflow
represented by structured aggregate metadata and with actionable groups given
priority over lower dispositions.
### Durable Files
Keep one canonical home for each class:
- `warnings.json` should contain versioned, grouped actionable warnings only;
- a new `diagnostics.json` should contain versioned advisory and observation
groups only, avoiding duplication of warning groups;
- `index.json` should link both files;
- `rejected.json` and manifest validation summaries should retain their current
separate responsibilities; and
- debug bundles should retain candidate-attempt detail plus the final grouped
projections.
This is preferable to keeping all detail in `warnings.json` and filtering only
the CLI: downstream consumers would otherwise continue to receive a semantically
mixed warning contract, and routine observations would still dominate the
durable file.
### CLI And Receipt
For a successful human run with actionable warnings, print a concise summary
such as:
```text
notarius: run completed with 2 warning groups (7 occurrences); details=/.../warnings.json
```
Advisories and observations should not produce the warning line. Their durable
path remains discoverable through `index.json`; a concise non-warning count can
be added to the ordinary success line only if operator testing shows value. An
ordinary successful run with no process degradation should write nothing to
the warning stream even when it publishes quality advisories or normalization
observations.
Create `notarius.run-result.v2` rather than redefining v1. It should expose at
least:
- `warning_group_count`;
- `warning_occurrence_count`; and
- `diagnostic_group_count` for non-warning durable groups.
The receipt should continue to expose validation status, validation summaries,
and rejected-output count independently. Process exit behavior should not
change as part of warning presentation reform.
### Checkpoint Semantics
Store the structured terminal diagnostic groups with accepted checkpoints and
replay them exactly once at their logical stage. Fresh and reused runs should
produce the same public groups and counts. Checkpoint events and debug records,
not diagnostic identity, should disclose whether computation was reused.
### Output Encoder Boundary
Prefer removing `OutputResult.Warnings`. A successful output encoder should
either return the complete logical files or fail. If future encoders genuinely
need to produce durable post-encoding warnings, introduce an explicit
two-phase prepare/finalize contract so those warnings can be included in the
same published collection. Do not retain the current self-inconsistent
one-pass capability.
## Resolution Of Required Design Questions
| Question | Recommendation | Viable alternative and tradeoff |
| --- | --- | --- |
| Explicit severity/disposition or external reason mapping? | Put validated disposition and category in the contract. | A central reason-code registry avoids payload fields but makes new modules depend on a second synchronized policy table and leaves downstream meaning implicit. |
| Keep all detail in `warnings.json` or separate it? | Separate grouped actionable warnings from grouped advisories/observations in `diagnostics.json`. | Keep the flat durable list and aggregate only CLI output; simpler migration, but it preserves the noisy downstream contract and ambiguous count. |
| Who owns aggregation? | Framework runner/coordinator after origin enrichment. | Output module aggregation keeps framework types smaller but duplicates policy across encoders and cannot repair missing validator origin. |
| Stable aggregation key? | Disposition, category, reason code, and full producer origin; exclude chunk/scope/message. | Explicit producer-supplied grouping keys offer flexibility but add another identity that can drift from reason codes. Message-template grouping is brittle and unsafe. |
| Samples and omissions? | Exact occurrence count plus deterministic bounded samples and numeric omitted-sample count. | Omission warning records preserve the current representation but inflate group counts and require prose parsing. |
| `warning_count` semantics? | Version receipt and replace ambiguity with group and occurrence counts. | Keep v1 count as published record length and add optional fields; compatible, but two competing warning counts remain easy to misuse. |
| Which normalization changes remain warnings? | Only exhausted process fallback. Unresolved membership is a data-quality advisory; successful canonicalization, reordering, ID repair, source-ref dedupe, and duplicate consolidation are observations. | Treat unresolved membership or semantic duplicate consolidation as warnings because they affect grounding or cardinality; more conservative, but it violates the process-only warning rule and reports accepted artifact quality as an operational failure. |
| Source-relatedness disposition? | Grouped advisory by default; preserve current approve behavior. | Retain warning disposition or make rejection configurable. Rejection requires production precision evidence; current lexical rules are not strong enough. |
| Checkpoint-loaded warnings? | Present the same logical groups as fresh execution and use checkpoint events for reuse provenance. | Mark groups as replayed; aids forensics but fragments aggregation and makes semantically equivalent runs differ. |
| ADR and schema versions? | Add an ADR and version the run receipt and diagnostic files. | Treat the work as CLI-only presentation and avoid an ADR; insufficient because module contracts, output files, checkpoint payloads, and downstream fields change. |
## Compatibility, Documentation, And ADR Implications
The target alters public and internal contracts enough to require a new ADR.
It should record:
- the distinction among warnings, advisories, observations, rejections, and
errors;
- the invariant that warnings are process-level signals, LLM-judged extraction
quality is never a warning, and ordinary non-degraded success has zero
warnings;
- module semantic ownership versus framework origin/aggregation ownership;
- bounded group and sample semantics;
- fresh/checkpoint equivalence; and
- the output-encoder decision.
Implementation should introduce `notarius.run-result.v2`. The grouped warning
and diagnostic envelopes should each carry their own schema version. Because
the content of `warnings.json` changes incompatibly from a flat array wrapper
to groups, release notes and the published JSON integration contract must call
out the migration. `index.json` gains the diagnostic file path.
Canonical documentation updates belong in:
- `docs/cli.md` for stderr presentation only;
- `docs/operations.md` for operator review and debug workflow;
- `docs/integrations/json-output.md` for warning and diagnostic file schemas;
- `docs/integrations/run-result.md` for v2 fields and compatibility;
- `docs/consumers/subprocess.md` and `docs/consumers/dnd-pipeline.md` for
downstream policy checks;
- `docs/internal/pipeline.md` for promotion, aggregation, retry, and checkpoint
mechanics;
- `docs/internal/modules.md` and `docs/internal/dnd.md` for producer rules and
the D&D classification matrix; and
- `docs/policy/architecture.md` for the durable ownership invariant after the
ADR is accepted and implemented.
No configuration knob is required for the first implementation. A fixed,
well-documented taxonomy is easier to reason about than per-reason display
overrides. Configurable escalation or suppression can be considered only after
production review demonstrates a concrete operator need.
## Test Coverage Assessment
Existing coverage worth preserving includes:
- accepted-attempt and terminal-rejection warning promotion;
- validator failure retry exhaustion and `warn_continue`;
- module semantic retry fallback;
- deterministic warning order under concurrent lane completion;
- chunk-plan invalidation and discarded-cache warning behavior;
- fresh/checkpoint warning equivalence;
- local D&D warning caps and safe dynamic-message quoting;
- JSON warning-file publication; and
- CLI stderr, debug, and receipt counts.
Material gaps are:
- no bound test for NPC- or spell-relatedness warnings;
- no generic warning-field or result-size validation;
- no test for output encoder warnings versus published `warnings.json`;
- no operator-level aggregation or bounded-sample tests;
- no fresh/resume test for grouped counts because groups do not yet exist; and
- no production evaluation of advisory precision.
Tests should protect the semantic relationships: exact occurrence counts,
bounded samples, deterministic group order, actionable-only warning
presentation, and cross-surface equality. They should not assert one exact
warning count for every complete D&D run or treat message wording as a public
API unless the wording itself enforces a security boundary.
## Audit Conclusion
The application is in a good position for warning reform. Its retry,
validation, checkpoint, and concurrency mechanics provide reliable points at
which to attach structured diagnostics. The most valuable change is not to
suppress individual reason codes; it is to replace the semantically flat,
origin-free collection with bounded typed groups and to reserve the word
“warning” for conditions that merit operator attention.
Provider-backed runs would improve prioritization and help tune the D&D
advisories, but they are not necessary to conclude that routine normalization
and heuristic doubt should not dominate stderr or the durable warning
contract. They should be gathered before changing heuristic acceptance policy
or adopting a numerical production warning-volume target.

View File

@@ -5,13 +5,64 @@ configuration, operations, internal, and integration docs. This roadmap records
future work only. Items are ordered roughly by current value and specificity,
not as committed release dates.
## Near-Term Validation And LLM Reliability
PromptKit now owns structural output repair within one completion. Notarius
owns stage candidates, validator chains, semantic rejection policy, bounded
feedback-aware stage retries, validation provenance, and reusable-state
eligibility, and the separation of actionable process warnings from quality
diagnostics. The remaining near-term work applies those completed foundations
to domain review and empirical evaluation.
### D&D Combat Scene Semantic Validation
- Add an optional production LLM-backed D&D validator that determines whether
proposed scene boundaries and classifications represent substantive active
combat correctly. Its central quality goal is that active combat is kept in
coherent scenes classified as `combat`, rather than split incorrectly or
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
- Resolve the validator's exact target before implementation. The current
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
`recap`, and `meta` classification. The preferred initial placement is
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
can compare one proposed kind with the corresponding transcript chunk.
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
question that can be answered from the complete transcript and proposed
range map, such as whether one continuous combat was fragmented across
inappropriate scene boundaries. Do not duplicate the same classification
judgment at both stages. Moving classification into chunk-plan annotations
would change the deliberately minimal, annotation-free chunk contract and
requires an explicit architecture review before it is selected.
- Validate both false negatives and false positives: a non-combat kind must not
omit substantive active combat, and a combat kind must be supported by such
combat. Keep the existing deterministic downstream rule that combat-turn
extraction runs only for an exact `combat` scene classification; semantic
review improves the upstream classification but does not replace that gate.
- Run the semantic validator through PromptKit, use a minimal required-field
structured response schema, and let PromptKit repair structural validator
output within its bounded budget. A contract-invalid final validator response
is a validator execution failure, not a semantic rejection and not a reason
to recursively validate the validator.
- Evaluate the prompt and decision policy against a small human-reviewed set
containing combat setup, active turns, interruptions, multi-phase encounters,
brief rules discussion, aftermath, recalled combat, and false-positive
hostile dialogue. Measure false acceptance, false rejection, retry success,
added calls, latency, and token cost before placing it in the production
default chain.
- An ADR is not required if classification remains owned by
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
Create or supersede an ADR if the work transfers scene classification into
the chunker or otherwise changes stage ownership or the durable chunk-plan
contract.
## Near-Term D&D Pipeline
### Evaluate Spell Extraction And Normalization
- Evaluate ordinary extraction retries and the completed normalization path
against a human-reviewed transcript set before adding repair-aware retries or
an LLM-backed semantic validator.
against a human-reviewed transcript set before and after adopting the shared
PromptKit repair and Notarius validation-retry policies above.
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
validator, and normalizer development. Treat model-quality review as an
iterative human evaluation aid, not a deterministic correctness gate.
@@ -24,28 +75,51 @@ not as committed release dates.
## Shared Normalization And Quality Work
### Generic LLM-Assisted Deduplication
The implemented source-backed core and initial D&D registry adoption are
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
and
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
The sections below keep broader extensions deferred.
- Add a reusable normalizer that asks an LLM to identify duplicate sets in a
list and propose one replacement element for each set.
- Define the minimum domain-neutral input contract, initially an ordered list
whose elements retain stable unique IDs as internal deterministic state.
Model proposals use contextual descriptors, or a specifically justified
request-local short label, rather than durable IDs. Artifact-kind
registrations or adapters may expose that structure without moving domain
rules into the generic package.
- Keep mutation deterministic: parse and validate the model's duplicate groups,
resolve every supplied descriptor or local label exactly, reject overlapping
or malformed groups, prevent unrelated insertion or deletion, and apply only
approved replacement operations in code.
- Preserve provenance needed for audit and downstream validation, and emit
warnings describing every collapsed group.
- Evaluate batching and context-window limits before applying the normalizer to
large artifact collections.
### Large-Collection Semantic Reconciliation
The model may use its own domain knowledge to judge semantic duplication; the
generic implementation is responsible only for the common proposal contract,
safety checks, and deterministic application of accepted changes.
- Evaluate deterministic candidate blocking only after representative registry
inputs exceed the active roadmap's bounded single-request limits. Blocking
should use cheap, explainable signals to form plausible comparison sets while
preserving the possibility that a duplicate appears outside a lexical name
match.
- Define correctness for candidates that appear in more than one block,
conflicting canonical selections, transitive identity across blocks, retry
isolation, and deterministic final ordering before implementation.
- Prefer a reconciliation graph or union plan with explicit conflict checks
over arbitrary fixed-size slices. Never silently treat a batch boundary as
evidence that two candidates are distinct.
- Record per-request bounds, block provenance, model calls, discarded
proposals, and final group derivation well enough to audit a collapse.
### Operator-Selected Semantic Policies
- Consider allowing an operator to select an approved semantic-policy prompt
for a typed reconciliation module without replacing the shared protocol,
response schema, or deterministic safety rules.
- Define the trusted asset source, configuration syntax, compatibility checks,
startup validation, provenance, prompt fingerprinting, checkpoint effects,
and support boundary before exposing the option.
- Prefer selection among registered, typed-policy-compatible prompt assets over
arbitrary filesystem prompt paths. Do not add this flexibility until an
operator workflow requires it; artifact-family-owned policy remains simpler
and safer for the initial implementation.
### Broader Reconciliation Inputs And Module Selection
- Revisit alternate context providers when a concrete non-source-backed entity
collection needs semantic reconciliation. Any extension must preserve the
same request-local identity, deterministic proposal validation, provenance,
and typed application guarantees.
- Consider a selectable generic normalizer only if Notarius gains a real
domain-neutral typed artifact contract that can safely support it. Do not
weaken exact artifact registration or introduce reflection-based arbitrary
JSON mutation merely to expose a universal module key.
### Validation And Review
@@ -102,6 +176,18 @@ checkpoint reuse, when an older artifact may be decoded or adapted, and when a
producer or all dependents must be recomputed. Do not add a general migration
framework until an actual contract change requires one.
### Artifact-family-oriented physical packaging
[ADR-0004](../adr/0004-package-modules-by-domain.md) currently groups production
extensions by domain and then by pipeline stage. After artifact-family
ownership terminology is established and more families span extraction,
normalization, validation, codecs, references, and assets, reassess whether a
feature-first physical layout would improve navigation and reduce scattered
changes enough to justify a repository-wide package migration. Any change must
address Go dependency cycles, registrar ownership, stable public module keys,
and supersession of the affected ADR-0004 decision. Conceptual artifact-family
ownership does not by itself require this move.
## Blue-Sky Platform And Operations
These ideas are intentionally less specified. Promote one into an earlier
@@ -117,7 +203,6 @@ section only after a concrete workflow, contract, and priority emerge.
### Distribution And Operations
- Packaged release artifacts for alpha distribution.
- A documented versioning and release process.
- Optional generated example-output fixtures with a regeneration procedure.
- Additional diagnostics or reporting views.

7
go.mod
View File

@@ -3,9 +3,14 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5
require (
gitea.maximumdirect.net/eric/promptkit v0.5.0
gitea.maximumdirect.net/eric/promptkit v0.9.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)
require golang.org/x/text v0.40.0
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
)

8
go.sum
View File

@@ -1,5 +1,9 @@
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

View File

@@ -0,0 +1,36 @@
// Package buildinfo resolves the product version embedded in a Notarius build.
package buildinfo
import (
"fmt"
"regexp"
"runtime/debug"
)
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
// Override is set at link time for controlled builds.
var Override string
// Version returns the release version embedded in the build, or development
// when the build does not carry a stable release tag.
func Version() (string, error) {
buildVersion := ""
if info, ok := debug.ReadBuildInfo(); ok {
buildVersion = info.Main.Version
}
return resolve(Override, buildVersion)
}
func resolve(override, buildVersion string) (string, error) {
if override != "" {
if !stableVersion.MatchString(override) {
return "", fmt.Errorf("build version override is not a stable release tag")
}
return override, nil
}
if stableVersion.MatchString(buildVersion) {
return buildVersion, nil
}
return "development", nil
}

View File

@@ -0,0 +1,45 @@
package buildinfo
import "testing"
func TestResolve(t *testing.T) {
tests := []struct {
name string
override string
buildVersion string
want string
wantErr bool
}{
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
{name: "development build", buildVersion: "(devel)", want: "development"},
{name: "missing build information", want: "development"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := resolve(tt.override, tt.buildVersion)
if tt.wantErr {
if err == nil {
t.Fatal("resolve() error = nil, want error")
}
return
}
if err != nil {
t.Fatalf("resolve() error = %v", err)
}
if got != tt.want {
t.Fatalf("resolve() = %q, want %q", got, tt.want)
}
})
}
}

View File

@@ -21,6 +21,10 @@ import (
const assembledSpellExtractorKey = "test/dnd/spell-casts"
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
@@ -67,28 +71,39 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
}
wantWarningReasons := []string{
wantDiagnosticReasons := []string{
spellnormalize.ReasonCodeSpellNameCanonicalized,
spellnormalize.ReasonCodeSourceReferencesNormalized,
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
"spell_not_near_source",
}
gotWarningReasons := make([]string, len(output.Warnings))
for index, warning := range output.Warnings {
gotWarningReasons[index] = warning.ReasonCode
gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
for index, group := range output.Diagnostics.Groups {
gotDiagnosticReasons[index] = group.ReasonCode
}
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
}
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
}
warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Truncated bool `json:"truncated"`
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
}
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
@@ -101,6 +116,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
}
}
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
}
correction := extractor.correctionSnapshot()
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
}
}
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
var normalizeChain *pipeline.ResolvedValidatorChain
@@ -127,15 +166,16 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
}
for _, warning := range output.Warnings {
if warning.ReasonCode == "spell_not_near_source" {
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
for _, group := range output.Diagnostics.Groups {
if group.ReasonCode == "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
}
}
}
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
@@ -161,12 +201,12 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
t.Fatalf("warnings = %#v, want terminal normalize catalog warning", output.Warnings)
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
}
}
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) {
func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
@@ -190,14 +230,20 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
}
warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings)
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
}
}
@@ -206,6 +252,47 @@ type assembledSpellPipelineOptions struct {
unknownSpell bool
}
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
extractor := &assembledCorrectingSpellExtractor{}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledCorrectingSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register correcting extractor: %v", err)
}
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
return assembledDirectSpellValidator{}, nil
}); err != nil {
t.Fatalf("register direct spell validator: %v", err)
}
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
extract.Retries = 1
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
},
Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return components.registries, resolved, extractor
}
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
@@ -251,6 +338,69 @@ type assembledSpellExtractor struct {
unknownSpell bool
}
type assembledCorrectingSpellExtractor struct {
mu sync.Mutex
correction *contracts.SemanticCorrection
}
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if req.Source == nil || req.Chunk == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
}
response := `{"spell":"accepted"}`
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
if req.Chunk.Index == 0 && req.Correction == nil {
response = `{"spell":"Mysterious Burst"}`
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
if req.Chunk.Index == 0 && req.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
e.mu.Lock()
e.correction = correction
e.mu.Unlock()
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
}
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
e.mu.Lock()
defer e.mu.Unlock()
correction, err := contracts.CloneSemanticCorrection(e.correction)
if err != nil {
return nil
}
return correction
}
type assembledDirectSpellValidator struct{}
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
for _, cast := range req.Value.SpellCasts {
if cast.Spell == "Mysterious Burst" {
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
}
}
return contracts.ValidationResult{Approved: true}, nil
}
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
@@ -296,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...)
}
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount
}
return count
}
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
t.Helper()
for _, file := range files {

View File

@@ -8,6 +8,8 @@ import (
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
)
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
}
}
func TestCommandVersionOutput(t *testing.T) {
previous := buildinfo.Override
t.Cleanup(func() { buildinfo.Override = previous })
tests := []struct {
name string
override string
wantCode int
wantStdout string
wantStderr string
}{
{name: "development", wantStdout: "notarius development\n"},
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buildinfo.Override = tt.override
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
explicit := writeCommandConfig(t, "explicit", "alpha")
environment := writeCommandConfig(t, "environment", "beta")

View File

@@ -189,10 +189,18 @@ func TestMaintainedCompleteExamplePublishesRegistryBackedEntityOccurrences(t *te
}
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
for _, laneID := range []string{"enemy-events", "npc-registry", "npc-occurrences", "item-registry", "item-occurrences", "location-registry", "location-occurrences"} {
if !containsString(evidence.SelectedLanes, laneID) || !evidenceHasLane(evidence, laneID) {
t.Fatalf("evidence context = %#v, want direct %s evidence", evidence, laneID)
if len(evidence) == 0 {
t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
}
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
for _, unit := range evidence {
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
}
if _, exists := seenEvidenceUnits[unit.ID]; exists {
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
}
seenEvidenceUnits[unit.ID] = struct{}{}
}
requests := client.requestsFor(enemyevents.PromptID)
@@ -388,8 +396,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
if err := json.Unmarshal(content, output); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
}
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock()
client.requests = append(client.requests, request)
client.requests = append(client.requests, snapshot)
client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
}
@@ -400,7 +412,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
var requests []contracts.StructuredCompletionRequest
for _, request := range client.requests {
if request.PromptID == promptID {
requests = append(requests, request)
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
}
}
return requests
@@ -415,17 +431,6 @@ func containsString(values []string, want string) bool {
return false
}
func evidenceHasLane(value evidencecontext.Document, laneID string) bool {
for _, context := range value.Contexts {
for _, reference := range context.EvidenceRefs {
if reference.LaneID == laneID {
return true
}
}
}
return false
}
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
for _, binding := range bindings {
if binding.SlotName == slotName && binding.Artifact != nil {

View File

@@ -91,8 +91,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if !reflect.DeepEqual(durable, want) {
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
}
if len(output.Warnings) != 0 {
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
}
}

View File

@@ -239,10 +239,10 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
}
warnings := readProductionJSON[struct {
Warnings []json.RawMessage `json:"warnings"`
Groups []json.RawMessage `json:"groups"`
}](t, filepath.Join(runRoot, "warnings.json"))
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
}
}

View File

@@ -19,12 +19,15 @@ import (
"testing/fstest"
"time"
"gopkg.in/yaml.v3"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
@@ -311,6 +314,81 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
t.Fatalf("prepare production scene and spell modules: %v", err)
}
schemaFS, err := components.assets.SchemaFS()
if err != nil {
t.Fatalf("production schema assets: %v", err)
}
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
t.Fatalf("generic reconciliation schema asset: %v", err)
}
options, err := components.assets.PromptKitOptions()
if err != nil {
t.Fatalf("production PromptKit options: %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
})))
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
if err != nil {
t.Fatalf("production prompt engine: %v", err)
}
promptFS, err := components.assets.PromptFS()
if err != nil {
t.Fatalf("production prompt assets: %v", err)
}
type manifest struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
Inputs []struct {
Name string `yaml:"name"`
} `yaml:"inputs"`
Messages []struct {
Role string `yaml:"role"`
} `yaml:"messages"`
}
preparedPrompts := 0
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
return nil
}
data, err := fs.ReadFile(promptFS, path)
if err != nil {
return err
}
var prompt manifest
if err := yaml.Unmarshal(data, &prompt); err != nil {
return err
}
for _, message := range prompt.Messages {
if message.Role != promptkit.RoleSystem && message.Role != promptkit.RoleUser {
return fmt.Errorf("production prompt %q uses role %q, want system or user", prompt.ID, message.Role)
}
}
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
for _, input := range prompt.Inputs {
inputs[input.Name] = promptkit.Inline(`{}`)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
})
if err != nil {
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
}
if prepared.OutputContract.RepairAttempts != 1 {
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
}
preparedPrompts++
return nil
}); err != nil {
t.Fatal(err)
}
if preparedPrompts == 0 {
t.Fatal("prepared no production prompts")
}
}
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
@@ -774,10 +852,10 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
}
warnings := readProductionJSON[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Warnings)
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Groups)
}
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
@@ -1021,8 +1099,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
}
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock()
client.requests = append(client.requests, req)
client.requests = append(client.requests, snapshot)
client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
}
@@ -1033,7 +1115,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
var requests []contracts.StructuredCompletionRequest
for _, req := range client.requests {
if req.PromptID == promptID {
requests = append(requests, req)
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
}
}
return requests

View File

@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
profileID: "malformed-profile",
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
},
{
name: "invalid profile source",
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
}
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
for _, profiles := range []string{
"id: child\nbase_profile: missing\n",
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
} {
t.Run("invalid inheritance", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
t.Fatal(err)
}
profileID := "child"
if strings.Contains(profiles, "id: first") {
profileID = "first"
}
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
t.Fatalf("profile preflight error = %v", err)
}
})
}
}

View File

@@ -16,6 +16,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
@@ -31,6 +32,7 @@ import (
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
const usage = `Usage:
notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
@@ -62,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
}
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
if len(args) > 0 && args[0] == "--version" {
if len(args) != 1 {
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
return 2
}
version, err := buildinfo.Version()
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "notarius %s\n", version)
return 0
}
var err error
opts, err = normalizeOptions(opts)
if err != nil {
@@ -359,7 +375,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
@@ -446,7 +462,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Warnings: referenceWarnings,
Diagnostics: referenceDiagnostics,
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
ChunkPlans: chunkPlans,
Checkpoints: checkpointRecorder,
@@ -455,7 +471,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
})
commandState.observeOutput(output)
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
commandState.observeOutput(output, diagnosticProjection)
}
if err != nil {
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
if output.Manifest.PipelineID != "" {
@@ -463,15 +482,21 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
}
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
}
var encodedResult []byte
if *machineOutput {
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -497,12 +522,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
}
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount)
if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok {
fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile))
}
fmt.Fprintln(stderr)
}
return 0
}
func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) {
for _, file := range files {
if file.Name == name {
return file.Name, true
}
}
return "", false
}
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
if summary == nil {
return nil
@@ -522,7 +560,7 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
return err
}
}
if err := summary.WriteWarnings(output.Warnings); err != nil {
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
return err
}
return summary.WriteCheckpointEvents(output.CheckpointEvents)

View File

@@ -590,10 +590,11 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.includeWarnings = true
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
harness.includeWarningFile = true
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
@@ -601,11 +602,12 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
if err != nil || !strings.Contains(string(output), "contract-warning") {
t.Fatalf("durable output = %q, %v", output, err)
}
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
bundle := onlyChildDir(t, roots.debug)
var warnings []contracts.Warning
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
t.Fatalf("debug warnings = %#v", warnings)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
t.Fatalf("debug diagnostics = %#v", diagnostics)
}
}

View File

@@ -48,8 +48,15 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != want {
t.Fatalf("%s mode = %#o, want %#o", path, info.Mode().Perm(), want)
mode := info.Mode().Perm()
if mode&^want != 0 {
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
}
if info.IsDir() && mode&0o700 != 0o700 {
t.Fatalf("%s mode = %#o, want owner access", path, mode)
}
if !info.IsDir() && mode != want {
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
}
}
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))

View File

@@ -7,25 +7,40 @@ import (
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const runResultSchemaVersion = "notarius.run-result.v1"
const runResultSchemaVersion = "notarius.run-result.v2"
type runResult struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status"`
DebugDirectory string `json:"debug_directory,omitempty"`
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningGroupCount int `json:"warning_group_count"`
WarningOccurrenceCount int `json:"warning_occurrence_count"`
DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
}
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if err != nil {
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
}
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
}
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
if strings.TrimSpace(output.Manifest.RunID) == "" {
return runResult{}, fmt.Errorf("run result requires a run ID")
}
@@ -51,14 +66,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
}
result := runResult{
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus,
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningGroupCount: len(diagnosticProjection.Warnings),
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
}
if strings.TrimSpace(debugDirectory) != "" {
@@ -85,6 +105,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
return result, nil
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
}
func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result)
if err != nil {

View File

@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_count"]; got != float64(0) {
t.Fatalf("warning_count = %v", got)
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
@@ -63,19 +63,19 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
if err != nil {
t.Fatal(err)
}
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err)
}

View File

@@ -52,8 +52,11 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := decoded["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", decoded)
}
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
}
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got)
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
}
}
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
output := testRunOutput()
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
if err != nil {
t.Fatal(err)
}
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
t.Fatalf("encoded result = %s", encoded)
}
}
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
tests := []struct {
name string
@@ -159,8 +182,15 @@ func testRunOutput() pipeline.RunOutput {
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Warnings: []contracts.Warning{{}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryFallback,
ReasonCode: "fallback",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}
}

View File

@@ -6,6 +6,7 @@ import (
"io"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -33,13 +34,17 @@ func (s *pipelineCommandState) setDebugPath(debugPath string) {
}
}
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput, diagnostics contracts.DiagnosticProjection) {
if s == nil {
return
}
s.report.OutputCount = len(output.NormalizeOutputs)
s.report.RejectedCount = len(output.Rejected)
s.report.WarningCount = len(output.Warnings)
s.report.WarningGroupCount = len(diagnostics.Warnings)
s.report.WarningOccurrenceCount = diagnostics.WarningOccurrenceCount
s.report.DiagnosticGroupCount = len(diagnostics.Diagnostics)
s.report.DiagnosticOccurrenceCount = diagnostics.DiagnosticOccurrenceCount
s.report.DiagnosticsTruncated = output.Diagnostics.Truncated
s.report.ValidationStatus = output.Manifest.ValidationStatus
}

View File

@@ -238,7 +238,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
},
}
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
t.Fatal(err)
}

View File

@@ -19,12 +19,12 @@ import (
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
const retries = 2
tests := []struct {
name string
responses []string
wantCalls int
wantRejected bool
wantSpell string
wantWarningCode string
name string
responses []string
wantCalls int
wantRejected bool
wantSpell string
wantAdvisoryCode string
}{
{
name: "unknown spell remains rejected after exhaustion",
@@ -42,9 +42,9 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
productionSpellResponse("Unknown Spell"),
productionSpellResponse("Aegis of Emberfall"),
},
wantCalls: 2,
wantSpell: "Aegis of Emberfall",
wantWarningCode: "spell_not_near_source",
wantCalls: 2,
wantSpell: "Aegis of Emberfall",
wantAdvisoryCode: "spell_not_near_source",
},
}
@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
t.Fatalf("materialize production references: %v", err)
}
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
llmClient := &catalogRetryLLMClient{responses: tt.responses}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
}
if len(output.Warnings) != 0 {
t.Fatalf("warnings = %#v, want no emitted warnings from rejected attempts", output.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
}
return
}
@@ -107,8 +108,16 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
}
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode {
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings)
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
}
if len(output.Diagnostics.Groups) != 2 {
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
}
for _, diagnostic := range output.Diagnostics.Groups {
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
}
}
})
}

View File

@@ -349,7 +349,7 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID)
}
report := readStateTestRunReport(t, debugPath)
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "approved" {
t.Fatalf("success report = %#v", report)
}
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
@@ -431,7 +431,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
bundlePath := onlyChildDir(t, roots.debug)
runID := filepath.Base(bundlePath)
report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != tc.wantValidation {
t.Fatalf("failure report = %#v", report)
}
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
@@ -445,7 +445,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")}
harness.extractErr = errors.New("synthetic partial pipeline failure")
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
@@ -454,7 +454,7 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
}
bundlePath := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningGroupCount != 1 || report.WarningOccurrenceCount != 1 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "failed" {
t.Fatalf("partial failure report = %#v", report)
}
@@ -463,10 +463,10 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
t.Fatalf("partial manifest = %#v", manifest)
}
var warnings []contracts.Warning
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
t.Fatalf("partial warnings = %#v", warnings)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundlePath, "final-diagnostics.json", &diagnostics)
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" {
t.Fatalf("partial diagnostics = %#v", diagnostics)
}
var events []pipeline.CheckpointEvent
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
@@ -863,11 +863,12 @@ type stateTestHarness struct {
chunkCalls, extractCalls int
runIDCalls uint64
extractErr error
chunkWarnings []contracts.Warning
chunkDiagnostics []contracts.ProducerDiagnostic
moduleProfiles []string
sessionIDs []string
outputWarnings []contracts.Warning
outputDiagnostics contracts.DiagnosticCollection
includeWarnings bool
includeWarningFile bool
}
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
@@ -892,7 +893,7 @@ func (h *stateTestHarness) options() Options {
panic(err)
}
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil
}); err != nil {
panic(err)
}
@@ -931,7 +932,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
c.harness.mu.Lock()
c.harness.chunkCalls++
c.harness.mu.Unlock()
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil
}
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
@@ -999,20 +1000,28 @@ func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNor
}
type stateTestOutput struct {
harness *stateTestHarness
includeWarnings bool
harness *stateTestHarness
includeDiagnostics bool
}
func (o stateTestOutput) Key() string { return "test/output" }
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
o.harness.mu.Lock()
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics)
o.harness.mu.Unlock()
data := []byte("{\"ok\":true}\n")
if o.includeWarnings && len(req.Warnings) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
}
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
files := []contracts.OutputFile{{Name: "result.json", Bytes: data}}
if o.harness.includeWarningFile {
files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")})
}
return contracts.OutputResult{Files: files}, nil
}
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {
return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}}
}
type failingDebugRecorder struct{}

View File

@@ -93,17 +93,44 @@ type NormalizedOutputManifest struct {
}
type RejectedOutputManifest struct {
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Validation *ValidationSummary `json:"validation,omitempty"`
}
// ValidationSummary is the bounded, durable outcome of validating one
// producer result. It deliberately contains identities and stable codes, not
// model responses, corrective guidance, validator diagnostics, or payloads.
type ValidationSummary struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Status string `json:"status"`
RejectingValidators []string `json:"rejecting_validators,omitempty"`
ReasonCodes []string `json:"reason_codes,omitempty"`
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
ProducerAttemptCount int `json:"producer_attempt_count"`
TerminalAction string `json:"terminal_action"`
}
// CloneValidationSummary returns an independently owned durable summary.
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
return summary
}
type CheckpointDecisionManifest struct {
@@ -163,6 +190,7 @@ type RunManifest struct {
References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`

View File

@@ -116,6 +116,11 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output)
@@ -196,6 +201,11 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options)
}
@@ -204,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out
}
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {

View File

@@ -35,28 +35,40 @@ type FilePromptKitLocalBackendConfig struct {
}
type FilePipelineProfile struct {
LLMProfile *string `yaml:"llm_profile,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
llmProfileSet bool `yaml:"-"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
llmProfileSet bool `yaml:"-"`
}
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
return err
}
type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
"llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile")
if err != nil {
return err
}
*p = FilePipelineProfile(decoded)
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
if err != nil {
return err
}
p.ValidationPolicy = policy
}
_, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"]
@@ -147,12 +159,14 @@ type FileDebugConfig struct {
}
type fileModuleBinding struct {
Module string
LLMProfile string
Retries int
Options map[string]any
References map[string]fileReferenceSource
Validators pipeline.ValidatorOverride
Module string
LLMProfile string
StructuredOutputRepairAttempts *int
ValidationPolicy *pipeline.ValidationPolicyOverride
Retries int
Options map[string]any
References map[string]fileReferenceSource
Validators pipeline.ValidatorOverride
}
type fileReferenceSource struct {
@@ -245,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
b.Module = strings.TrimSpace(module)
return nil
case yaml.MappingNode:
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valueNode := node.Content[i+1]
if _, exists := seen[keyNode.Value]; exists {
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
}
seen[keyNode.Value] = struct{}{}
switch keyNode.Value {
case "module":
var module string
@@ -264,6 +283,18 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
if b.LLMProfile == "" {
return fmt.Errorf("llm_profile must not be empty when set")
}
case "structured_output_repair_attempts":
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
if err != nil {
return err
}
b.StructuredOutputRepairAttempts = attempts
case "validation_policy":
policy, err := parseValidationPolicy(valueNode, "module binding")
if err != nil {
return err
}
b.ValidationPolicy = policy
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
@@ -304,15 +335,102 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
return pipeline.ModuleBinding{
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References),
Validators: b.Validators,
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References),
Validators: cloneValidatorOverride(b.Validators),
}
}
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value == key {
return node.Content[i+1], true
}
}
return nil, false
}
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s validation_policy must be an object", context)
}
policy := &pipeline.ValidationPolicyOverride{}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
value := node.Content[i+1]
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
}
seen[key] = struct{}{}
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
}
switch key {
case "producer_structural_failure":
value := pipeline.ProducerStructuralFailureAction(value.Value)
policy.ProducerStructuralFailure = &value
case "semantic_rejection":
value := pipeline.SemanticRejectionAction(value.Value)
policy.SemanticRejection = &value
case "validator_failure":
value := pipeline.ValidatorFailureAction(value.Value)
policy.ValidatorFailure = &value
default:
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
}
}
if err := policy.Validate(); err != nil {
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
}
return policy, nil
}
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("%s must be an object", context)
}
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value != "structured_output_repair_attempts" {
continue
}
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
return err
}
}
return nil
}
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
if node.Tag == "!!null" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
}
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
}
var attempts int
if err := node.Decode(&attempts); err != nil {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
}
if attempts < 0 || attempts > 3 {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return &attempts, nil
}
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
if attempts == nil {
return nil
}
value := *attempts
return &value
}
func LoadFileConfig(path string) (FileConfig, error) {
data, err := os.ReadFile(path)
if err != nil {
@@ -518,11 +636,13 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return err
}
profile := pipeline.PipelineProfile{
ID: pipelineID,
LLMProfile: llmProfile,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References),
ID: pipelineID,
LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()

View File

@@ -138,6 +138,168 @@ pipelines:
}
}
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
%s
input: input
artifacts:
lane:
extract:
module: extract
structured_output_repair_attempts: 2
validators:
- module: validator
structured_output_repair_attempts: 3
`
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
}
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
}
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
}
})
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
profile := cfg.Pipelines["main"]
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
}
lane := profile.Artifacts["lane"]
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
}
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
}
fileProfile := file.Pipelines["main"]
fileLane := fileProfile.Artifacts["lane"]
*fileLane.Extract.StructuredOutputRepairAttempts = 1
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
fileProfile.Artifacts["lane"] = fileLane
file.Pipelines["main"] = fileProfile
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
}
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
}
cloned := cloneConfig(cfg)
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
}
redacted := cfg.Redacted()
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
}
summary := cfg.RedactedSummaryPayload().(Config)
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatal(err)
}
var roundTripped Config
if err := json.Unmarshal(data, &roundTripped); err != nil {
t.Fatal(err)
}
roundTrippedProfile := roundTripped.Pipelines["main"]
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
}
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
}
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
})
}
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
input: input
artifacts:
lane:
extract: extract
%s
`
const bindingYAML = `version: 4
pipelines:
main:
input:
module: input
%s
artifacts:
lane:
extract: extract
`
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
}
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
const configYAML = `version: 4
pipelines:

View File

@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = redactBinding(cloneModuleBinding(in.Input))
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)

View File

@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
}
}
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
semantic := pipeline.SemanticRejectionRejectOutput
validator := pipeline.ValidatorFailureFailRun
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
effective := EffectiveConfig{
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
"main": {ValidationPolicy: configured},
}},
ResolvedPipeline: pipeline.ResolvedPipeline{
ConfiguredValidationPolicy: configured,
ChunkValidationPolicy: pipeline.ValidationPolicy{
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
ValidatorFailure: pipeline.ValidatorFailureFailRun,
},
},
}
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
encoded, err := json.Marshal(payload)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
}
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatal("redacted payload aliases validation policy")
}
}
func redactionTestBinding(name string) pipeline.ModuleBinding {
return pipeline.ModuleBinding{
Module: "safe-" + name,

View File

@@ -116,6 +116,14 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
}
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
return err
}
if profile.ValidationPolicy != nil {
if err := profile.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
}
}
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err
}
@@ -195,6 +203,19 @@ func validateBinding(
binding pipeline.ModuleBinding,
referencesAllowed bool,
) error {
if binding.ValidationPolicy != nil {
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
}
if err := binding.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
}
}
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
return err
}
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
return err
}
@@ -219,6 +240,13 @@ func validateBinding(
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
}
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
if attempts != nil && (*attempts < 0 || *attempts > 3) {
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return nil
}
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set {
return nil
@@ -230,6 +258,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
}
for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
return err
}
if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context)
}
@@ -239,8 +270,11 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context)
if validator.ValidationPolicy != nil {
return fmt.Errorf("%s validation_policy is not supported", context)
}
if validator.Retries < 0 {
return fmt.Errorf("%s retries must be greater than or equal to zero", context)
}
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context)

View File

@@ -374,17 +374,17 @@ func TestValidateValidatorBindingRules(t *testing.T) {
want: "chunk validators[0] module must not be empty",
},
{
name: "validator retries",
name: "negative validator retries",
setup: func(profile *pipeline.PipelineProfile) {
profile.Chunk.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{{
Module: "validator",
Retries: 1,
Retries: -1,
}},
}
},
want: "chunk validators[0] retries are not supported",
want: "chunk validators[0] retries must be greater than or equal to zero",
},
{
name: "validator references",

View File

@@ -0,0 +1,97 @@
package config
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
const valid = `version: 4
pipelines:
main:
validation_policy:
producer_structural_failure: reject_output
semantic_rejection: fail_run
input: seriatim
chunk:
module: generic
validation_policy:
validator_failure: fail_run
artifacts:
lane:
extract:
module: extract
validation_policy:
semantic_rejection: reject_output
`
cfg := applyFileConfig(t, valid)
profile := cfg.Pipelines["main"]
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
}
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
}
lane := profile.Artifacts["lane"]
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
}
for _, test := range []struct {
name string
yaml string
}{
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
}
})
}
}
func TestValidationPolicyPlacementRules(t *testing.T) {
policy := &pipeline.ValidationPolicyOverride{}
semantic := pipeline.SemanticRejectionRejectOutput
policy.SemanticRejection = &semantic
base := pipeline.PipelineProfile{
ID: "main",
Input: pipeline.Binding("input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"lane": {Extract: pipeline.Binding("extract")},
},
}
for _, test := range []struct {
name string
mutate func(*pipeline.PipelineProfile)
}{
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
{"output", func(profile *pipeline.PipelineProfile) {
profile.Output = pipeline.Binding("output")
profile.Output.ValidationPolicy = policy
}},
{"validator", func(profile *pipeline.PipelineProfile) {
lane := profile.Artifacts["lane"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
profile.Artifacts["lane"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := base
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
test.mutate(&profile)
cfg := Default()
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
}
})
}
}

View File

@@ -110,7 +110,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
if err := summary.WriteRunReport(RunReport{RunID: bundle.RunID(), PipelineID: "test"}); err != nil {
t.Fatal(err)
}
if err := summary.WriteWarnings([]contracts.Warning{{ReasonCode: "test"}}); err != nil {
if err := summary.WriteDiagnostics(contracts.DiagnosticCollection{}); err != nil {
t.Fatal(err)
}
if err := summary.WriteError("failed"); err != nil {
@@ -126,7 +126,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
ArtifactRunManifest,
ArtifactChunkPlan,
ArtifactRunReport,
ArtifactWarnings,
ArtifactDiagnostics,
ArtifactErrorLog,
} {
info, err := os.Stat(filepath.Join(bundle.SummaryRoot(), name))

View File

@@ -20,7 +20,7 @@ const (
ArtifactRunManifest = "run-manifest.json"
ArtifactChunkPlan = "chunk-plan.json"
ArtifactRunReport = "run-report.json"
ArtifactWarnings = "warnings.json"
ArtifactDiagnostics = "final-diagnostics.json"
ArtifactErrorLog = "error.log"
)
@@ -45,15 +45,19 @@ type Invocation struct {
StartedAt time.Time `json:"started_at"`
}
type RunReport struct {
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputPath string `json:"output_path,omitempty"`
DebugPath string `json:"debug_path,omitempty"`
Succeeded bool `json:"succeeded"`
OutputCount int `json:"output_count"`
RejectedCount int `json:"rejected_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status,omitempty"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputPath string `json:"output_path,omitempty"`
DebugPath string `json:"debug_path,omitempty"`
Succeeded bool `json:"succeeded"`
OutputCount int `json:"output_count"`
RejectedCount int `json:"rejected_count"`
WarningGroupCount int `json:"warning_group_count"`
WarningOccurrenceCount int `json:"warning_occurrence_count"`
DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status,omitempty"`
}
type SummaryWriter struct {
root, runID string
@@ -101,8 +105,8 @@ func (w *SummaryWriter) WriteChunkPlan(v artifacts.ChunkPlanSummary) error {
return w.writeJSON(ArtifactChunkPlan, v)
}
func (w *SummaryWriter) WriteRunReport(v RunReport) error { return w.writeJSON(ArtifactRunReport, v) }
func (w *SummaryWriter) WriteWarnings(v []contracts.Warning) error {
return w.writeJSON(ArtifactWarnings, v)
func (w *SummaryWriter) WriteDiagnostics(v contracts.DiagnosticCollection) error {
return w.writeJSON(ArtifactDiagnostics, contracts.CloneDiagnosticCollection(v))
}
func (w *SummaryWriter) WriteError(message string) error {
return w.writeBytes(ArtifactErrorLog, []byte(message+"\n"))

View File

@@ -24,7 +24,6 @@ type filesystemCheckpointFixture struct {
merge pipeline.CheckpointArtifact
normalize pipeline.CheckpointArtifact
dependencies []pipeline.CheckpointFingerprint
warnings []contracts.Warning
rejected []contracts.RejectedOutput
}
@@ -36,9 +35,9 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
fixture.doc.Metadata["owner"] = "caller mutation"
fixture.extract.Artifact.Content[0] = 'x'
fixture.extract.Artifact.Metadata["content"] = "caller mutation"
fixture.extract.Diagnostics[0].Diagnostic.Samples[0].Message = "caller mutation"
fixture.merge.Artifact.Content[0] = 'x'
fixture.normalize.Artifact.Content[0] = 'x'
fixture.warnings[0].Message = "caller mutation"
fixture.rejected[0].Message = "caller mutation"
t.Run("source", func(t *testing.T) {
@@ -60,11 +59,11 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
t.Run("extract", func(t *testing.T) {
got, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Warnings) != 1 {
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Outputs[0].Diagnostics) != 1 {
t.Fatalf("extract result=%#v decision=%#v", got, decision)
}
output := got.Outputs[0]
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || got.Warnings[0].ReasonCode != "partial" || got.Rejected[0].ReasonCode != "invalid_source" {
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || output.Diagnostics[0].Diagnostic.ReasonCode != "normalized_record" || got.Rejected[0].ReasonCode != "invalid_source" {
t.Fatalf("extract values were not restored: %#v", got)
}
manifest := readManifest[ExtractLaneManifest](t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "manifest.json"))
@@ -73,37 +72,36 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
}
got.Outputs[0].Artifact.Content[0] = 'y'
got.Warnings[0].Message = "loaded mutation"
reloaded, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) || reloaded.Warnings[0].Message != "partial output" {
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) {
t.Fatalf("extract reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
}
})
for _, tt := range []struct {
name string
load func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision)
load func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision)
want []byte
}{
{name: "merge", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
{name: "merge", load: func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision) {
got, decision := fixture.loader.Merge("lane-a", "merge-module", fixture.dependencies)
return got.Output, got.Warnings, decision
return got.Output, decision
}, want: []byte(`{"spells":["fire"]}`)},
{name: "normalize", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
{name: "normalize", load: func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision) {
got, decision := fixture.loader.Normalize("lane-a", "normalize-module", fixture.dependencies)
return got.Output, got.Warnings, decision
return got.Output, decision
}, want: []byte(`{"spells":["fire"],"normalized":true}`)},
} {
t.Run(tt.name, func(t *testing.T) {
got, warnings, decision := tt.load()
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(warnings) != 1 || warnings[0].ReasonCode != "review" {
t.Fatalf("%s result=%#v warnings=%#v decision=%#v", tt.name, got, warnings, decision)
got, decision := tt.load()
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(got.Diagnostics) != 1 || got.Diagnostics[0].Diagnostic.ReasonCode != "normalized_record" {
t.Fatalf("%s result=%#v decision=%#v", tt.name, got, decision)
}
got.Artifact.Content[0] = 'z'
reloaded, warnings, decision := tt.load()
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) || warnings[0].Message != "review manually" {
t.Fatalf("%s reload changed after loaded mutation: %#v warnings=%#v decision=%#v", tt.name, reloaded, warnings, decision)
reloaded, decision := tt.load()
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) {
t.Fatalf("%s reload changed after loaded mutation: %#v decision=%#v", tt.name, reloaded, decision)
}
})
}
@@ -159,6 +157,7 @@ func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
}{
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"v2 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV2 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"v3 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV3 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, pipeline.CheckpointReasonIdentityMismatch},
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, pipeline.CheckpointReasonStageMismatch},
@@ -204,6 +203,11 @@ func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T)
{"content digest", func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other"
}, pipeline.CheckpointReasonArtifactDigestMismatch},
{"diagnostics", func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["diagnostics"] = []any{map[string]any{
"diagnostic": map[string]any{"disposition": "warning", "category": "configuration", "reason_code": "invalid", "occurrence_count": float64(0)},
}}
}, pipeline.CheckpointReasonArtifactPayloadInvalid},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
@@ -337,9 +341,6 @@ func TestFilesystemLoaderReadsAcceptedNormalizeWithoutStageDependencies(t *testi
if checkpoint.Output.Artifact.Content == nil || string(checkpoint.Output.Artifact.Content) != string(fixture.normalize.Artifact.Content) {
t.Fatalf("accepted normalize output = %#v, want recorded artifact", checkpoint.Output)
}
if len(checkpoint.Warnings) != 1 || checkpoint.Warnings[0].ReasonCode != "normalized" {
t.Fatalf("accepted normalize warnings = %#v", checkpoint.Warnings)
}
}
func TestFilesystemLoaderRejectsInvalidAcceptedNormalize(t *testing.T) {
@@ -402,8 +403,7 @@ func seedAcceptedNormalizeCheckpoint(t *testing.T) filesystemCheckpointFixture {
t.Fatal(err)
}
stepRecorder := recorder.(pipeline.StepCheckpointRecorder)
warnings := []contracts.Warning{{Scope: "normalize", ReasonCode: "normalized", Message: "normalized warning"}}
if err := stepRecorder.NormalizeSucceededForStep("step-1", "lane-a", "normalize-module", []pipeline.CheckpointFingerprint{{Name: "merge", Value: "sha256:unavailable"}}, artifact, warnings); err != nil {
if err := stepRecorder.NormalizeSucceededForStep("step-1", "lane-a", "normalize-module", []pipeline.CheckpointFingerprint{{Name: "merge", Value: "sha256:unavailable"}}, artifact); err != nil {
t.Fatal(err)
}
loader, err := NewFilesystemLoader(root, identity)
@@ -493,20 +493,18 @@ func seedFilesystemCheckpoints(t *testing.T) filesystemCheckpointFixture {
merge: checkpointArtifact("merge", `{"spells":["fire"]}`),
normalize: checkpointArtifact("normalize", `{"spells":["fire"],"normalized":true}`),
dependencies: []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:source"}, {Name: "chunk-plan", Value: "sha256:plan"}},
warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "partial", Message: "partial output"}},
rejected: []contracts.RejectedOutput{{Stage: "extract", LaneID: "lane-a", ModuleKey: "extract-module", ChunkID: "chunk-a", ValidatorName: "source_refs", ReasonCode: "invalid_source", Message: "source reference is invalid", AttemptCount: 1}},
}
if err := recorder.SourceSucceeded("source-module", &fixture.doc); err != nil {
t.Fatal(err)
}
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected, fixture.warnings); err != nil {
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected); err != nil {
t.Fatal(err)
}
mergeWarnings := []contracts.Warning{{Scope: "merge", ReasonCode: "review", Message: "review manually"}}
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge, mergeWarnings); err != nil {
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge); err != nil {
t.Fatal(err)
}
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize, mergeWarnings); err != nil {
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize); err != nil {
t.Fatal(err)
}
fixture.loader, err = NewFilesystemLoader(root, identity)
@@ -527,7 +525,8 @@ func checkpointArtifact(module, content string) pipeline.CheckpointArtifact {
return pipeline.CheckpointArtifact{
LaneID: "lane-a", ModuleKey: module, SourceID: "document-1", ChunkID: "chunk-a", ChunkIndex: 0,
ChunkRef: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
Diagnostics: []pipeline.CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionObservation, Category: contracts.DiagnosticCategoryNormalization, ReasonCode: "normalized_record", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fixture", Message: "record normalized"}}}}},
}
}

View File

@@ -86,7 +86,7 @@ func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, depe
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ExtractCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
}
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected)}, reusedDecision()
}
func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
@@ -112,7 +112,7 @@ func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, depend
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.MergeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
}
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.MergeCheckpoint{Output: values[0]}, reusedDecision()
}
func (l *FilesystemLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
@@ -138,7 +138,7 @@ func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, de
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
}
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.NormalizeCheckpoint{Output: values[0]}, reusedDecision()
}
func (l *FilesystemLoader) AcceptedNormalize(stepID, laneID, moduleKey string) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
@@ -163,7 +163,7 @@ func (l *FilesystemLoader) AcceptedNormalize(stepID, laneID, moduleKey string) (
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
}
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused)
return pipeline.NormalizeCheckpoint{Output: values[0]}, decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused)
}
func (l *FilesystemLoader) validateAcceptedNormalizeManifest(manifest StageManifest, stepID, laneID, moduleKey string) pipeline.CheckpointDecision {
@@ -205,11 +205,33 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactCodecIncompatible, err: fmt.Errorf("artifact codec identity is incomplete")}
}
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
diagnostics, err := cloneAndValidateCheckpointDiagnostics(v.Diagnostics)
if err != nil {
return nil, err
}
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}, Diagnostics: diagnostics})
}
return out, nil
}
func cloneAndValidateCheckpointDiagnostics(values []pipeline.CheckpointDiagnostic) ([]pipeline.CheckpointDiagnostic, error) {
if len(values) == 0 {
return nil, nil
}
diagnostics := make([]contracts.ProducerDiagnostic, len(values))
for index, value := range values {
diagnostics[index] = value.Diagnostic
}
if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil {
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactPayloadInvalid, err: fmt.Errorf("checkpoint diagnostics: %w", err)}
}
cloned := make([]pipeline.CheckpointDiagnostic, len(values))
for index, value := range values {
cloned[index] = pipeline.CheckpointDiagnostic{Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{value.Diagnostic})[0], ValidatorKey: value.ValidatorKey}
}
return cloned, nil
}
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLoadingDisabled)

View File

@@ -3,7 +3,8 @@ package checkpoint
import "time"
const (
WorkspaceSchemaVersion = "notarius.workspace.v3"
WorkspaceSchemaVersion = "notarius.workspace.v4"
WorkspaceSchemaVersionV3 = "notarius.workspace.v3"
WorkspaceSchemaVersionV2 = "notarius.workspace.v2"
WorkspaceSchemaVersionV1 = "notarius.workspace.v1"
)

View File

@@ -10,6 +10,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -80,18 +81,18 @@ func (r *FilesystemRecorder) ExtractRunningForStep(stepID, laneID string, module
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return r.ExtractSucceededForStep("", laneID, moduleKey, dependencies, outputs, rejected, warnings)
func (r *FilesystemRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return r.ExtractSucceededForStep("", laneID, moduleKey, dependencies, outputs, rejected)
}
func (r *FilesystemRecorder) ExtractSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
func (r *FilesystemRecorder) ExtractSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected)}
if err := r.writePayload(lanePayloadPath("extract", stepID, laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(StageExtract, statusForRejected(rejected), stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests(outputs))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.ValidationStatus = validationStatusString(rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
@@ -118,17 +119,17 @@ func (r *FilesystemRecorder) MergeRunningForStep(stepID, laneID string, moduleKe
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
return r.MergeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
func (r *FilesystemRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
return r.MergeSucceededForStep("", laneID, moduleKey, dependencies, output)
}
func (r *FilesystemRecorder) MergeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
func (r *FilesystemRecorder) MergeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
if err := r.writePayload(lanePayloadPath("merge", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output)}); err != nil {
return err
}
manifest := r.laneManifest(StageMerge, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.ValidationStatus = validationStatusString(nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
@@ -166,17 +167,17 @@ func (r *FilesystemRecorder) NormalizeRunningForStep(stepID, laneID string, modu
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
return r.NormalizeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
func (r *FilesystemRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
return r.NormalizeSucceededForStep("", laneID, moduleKey, dependencies, output)
}
func (r *FilesystemRecorder) NormalizeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
func (r *FilesystemRecorder) NormalizeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
if err := r.writePayload(lanePayloadPath("normalize", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output)}); err != nil {
return err
}
manifest := r.laneManifest(StageNormalize, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.ValidationStatus = validationStatusString(nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
@@ -248,39 +249,37 @@ type sourceDocumentEnvelope struct {
}
type binaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type artifactCheckpointEnvelope struct {
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content binaryEnvelope `json:"content"`
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content binaryEnvelope `json:"content"`
Diagnostics []pipeline.CheckpointDiagnostic `json:"diagnostics,omitempty"`
}
type artifactExtractEnvelope struct {
Outputs []artifactCheckpointEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type artifactSingleEnvelope struct {
Output artifactCheckpointEnvelope `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
Output artifactCheckpointEnvelope `json:"output"`
}
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata), Diagnostics: cloneCheckpointDiagnostics(output.Diagnostics)}
}
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
if len(outputs) == 0 {
@@ -292,6 +291,20 @@ func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifa
}
return out
}
func cloneCheckpointDiagnostics(values []pipeline.CheckpointDiagnostic) []pipeline.CheckpointDiagnostic {
if len(values) == 0 {
return nil
}
cloned := make([]pipeline.CheckpointDiagnostic, len(values))
for index, value := range values {
cloned[index] = pipeline.CheckpointDiagnostic{
Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{value.Diagnostic})[0],
ValidatorKey: value.ValidatorKey,
}
}
return cloned
}
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs {
@@ -300,13 +313,12 @@ func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.Che
return normalizeFingerprints(values)
}
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any) binaryEnvelope {
return binaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: contentDigest(content),
MediaType: mediaType,
Metadata: cloneMetadata(metadata),
Warnings: cloneWarnings(warnings),
}
}
@@ -333,18 +345,19 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
return out
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return nil
}
return append([]contracts.Warning(nil), warnings...)
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
return append([]contracts.RejectedOutput(nil), rejected...)
cloned := make([]contracts.RejectedOutput, len(rejected))
for index, item := range rejected {
cloned[index] = item
if item.Validation != nil {
summary := artifacts.CloneValidationSummary(*item.Validation)
cloned[index].Validation = &summary
}
}
return cloned
}
func cloneMetadata(metadata map[string]any) map[string]any {
@@ -452,13 +465,10 @@ func statusForRejected(rejected []contracts.RejectedOutput) StageStatus {
return StatusSucceeded
}
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
func validationStatusString(rejected []contracts.RejectedOutput) string {
if len(rejected) > 0 {
return "rejected"
}
if len(warnings) > 0 {
return "approved_with_warnings"
}
return "approved"
}

View File

@@ -17,7 +17,7 @@ func TestRootBasedRecorderOutputIsReusable(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := recorder.ExtractSucceeded("lane", "module", nil, nil, nil, nil); err != nil {
if err := recorder.ExtractSucceeded("lane", "module", nil, nil, nil); err != nil {
t.Fatal(err)
}
loader, err := NewFilesystemLoader(root, identity)
@@ -44,7 +44,7 @@ func TestStepAwareRecorderAndLoaderIsolateLaneState(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := recorder.(pipeline.StepCheckpointRecorder).ExtractSucceededForStep("step-a", "lane", "module", nil, nil, nil, nil); err != nil {
if err := recorder.(pipeline.StepCheckpointRecorder).ExtractSucceededForStep("step-a", "lane", "module", nil, nil, nil); err != nil {
t.Fatal(err)
}
loader, err := NewFilesystemLoader(root, identity)
@@ -87,7 +87,7 @@ func TestStepAwareCheckpointPreservesDistinctDotIdentities(t *testing.T) {
LaneID: "lane", ModuleKey: "normalize-module", SourceID: "source", ChunkID: "chunk", ChunkRef: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
Artifact: contracts.SerializedArtifact{Kind: "kind", Schema: contracts.ArtifactSchema{ID: "schema", Name: "Schema", Version: "1"}, MediaType: "application/json", Content: []byte(test.content)},
}
if err := stepRecorder.NormalizeSucceededForStep(test.stepID, "lane", "normalize-module", nil, artifact, nil); err != nil {
if err := stepRecorder.NormalizeSucceededForStep(test.stepID, "lane", "normalize-module", nil, artifact); err != nil {
t.Fatalf("record %q: %v", test.stepID, err)
}
}
@@ -119,7 +119,7 @@ func TestStepAwareCheckpointPreservesDistinctDotIdentities(t *testing.T) {
}
func TestCheckpointSchemaCompatibilityIdentifiers(t *testing.T) {
if WorkspaceSchemaVersion != "notarius.workspace.v3" || WorkspaceSchemaVersionV2 != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
if WorkspaceSchemaVersion != "notarius.workspace.v4" || WorkspaceSchemaVersionV3 != "notarius.workspace.v3" || WorkspaceSchemaVersionV2 != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
t.Fatal("checkpoint schema identifiers are incorrect")
}
}

View File

@@ -14,6 +14,7 @@ import (
"syscall"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -340,6 +341,9 @@ func validateRecord(record pipeline.ChunkPlanRecord, requestedDigest string) err
if record.SchemaVersion != SchemaVersion {
return fmt.Errorf("schema_version %q is not supported", record.SchemaVersion)
}
if err := contracts.ValidateProducerDiagnostics(record.Diagnostics); err != nil {
return fmt.Errorf("diagnostics: %w", err)
}
if _, err := digestPathSegment(requestedDigest); err != nil {
return err
}

View File

@@ -281,7 +281,7 @@ func TestFilesystemStoreReportsInvalidRecordsAsRecoverable(t *testing.T) {
return bytes.Replace(data, []byte(`{"schema_version"`), []byte(`{"SENTINEL_UNKNOWN_FIELD":true,"schema_version"`), 1)
}},
{name: "truncated JSON", mutate: func(data []byte) []byte { return data[:len(data)/2] }},
{name: "legacy v1 record", mutate: replaceJSON(`notarius.chunk-plan.v2`, `notarius.chunk-plan.v1`)},
{name: "legacy v2 record", mutate: replaceJSON(`notarius.chunk-plan.v3`, `notarius.chunk-plan.v2`)},
{name: "source mismatch", mutate: replaceJSON(testSourceDigest, "sha256:"+strings.Repeat("b", 64))},
{name: "plan digest mismatch", mutate: func(data []byte) []byte {
prefix := []byte(`"plan_digest":"sha256:`)
@@ -498,7 +498,13 @@ func testRecord(t *testing.T, value int) pipeline.ChunkPlanRecord {
References: []artifacts.ReferenceProvenance{{Stage: "chunk", SlotName: "guide", OriginType: "file", OriginURI: "file:///guide.txt", Digest: "sha256:reference"}},
Metadata: map[string]any{"prompt_id": "test/prompt", "enabled": true},
},
Warnings: []contracts.Warning{{Scope: "chunk/test", ReasonCode: "observed", Message: "warning"}},
Diagnostics: []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryConfiguration,
ReasonCode: "empty_reference",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "reference", Message: "Reference was empty."}},
}},
CreatedAt: time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC),
}
}

Some files were not shown because too many files have changed in this diff Show More