Compare commits
118 Commits
92e89076a2
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 61436d7c18 | |||
| 079d5af337 | |||
| 1025001f20 | |||
| da14924a02 | |||
| 2065a8288b | |||
| af0119cc1d | |||
| 54de2b816a | |||
| 4dbbf68051 | |||
| 480680b257 | |||
| 6a1fd7bdb6 | |||
| ccba2ce3f9 | |||
| 1f1967c8d2 | |||
| 5175cb0722 | |||
| ba569594a1 | |||
| acb04954eb | |||
| 610dd3d7c3 | |||
| e92bcfa74c | |||
| e6b2ae88d2 | |||
| b2e83bd6e7 | |||
| 7f01c3e79e | |||
| 7b5f4ebd42 | |||
| 4bb4582695 | |||
| 7d3434e5f6 | |||
| 29872b2e28 | |||
| b487d93186 | |||
| 8dd7a4324d | |||
| 04ba87e174 | |||
| a26d6ed042 | |||
| 759d32403f | |||
| 1a7b20c766 | |||
| 85a5b52be7 | |||
| 9d0faabf61 | |||
| 1c3da3e869 | |||
| 9abd93502f | |||
| c8a29a5fa2 | |||
| 916d9210fd | |||
| 3d3f16db4a | |||
| 63c397d86a | |||
| 9a92212632 | |||
| 0ef8931697 | |||
| e00cc45c6b | |||
| ab9b743df6 | |||
| 3bd3c7ebf7 | |||
| 75a3f51cee | |||
| 2be999ebd3 | |||
| 32fe7c5b98 | |||
| 55247c47ab | |||
| 5e5c69bf9d | |||
| b4a81f8b09 | |||
| adfed22e1a | |||
| 75b1e2f68b | |||
| 4473363d9f | |||
| 6eb45e0003 | |||
| 0585ad76dc | |||
| 56145c3b7e | |||
| a478fd86c5 | |||
| 916532100d | |||
| bef8ca263b | |||
| f6d037b613 | |||
| 449b506804 | |||
| 071a78ae22 | |||
| ad1cba41c2 | |||
| 67338798aa | |||
| e95e2f2220 | |||
| 628b8d1800 | |||
| d24d4609b6 | |||
| c7f79fb38e | |||
| 8c071800cf | |||
| 569e12c6f4 | |||
| 5b6eb591b2 | |||
| b630384aa0 | |||
| 297d58f090 | |||
| ee71dc4937 | |||
| 65e5d65d14 | |||
| b40b40aaf3 | |||
| f120be1cb4 | |||
| 17673d74ea | |||
| ef19a03cbf | |||
| 0546f6eb4f | |||
| d28d1062e0 | |||
| b3ebfcef37 | |||
| b70d9f77e3 | |||
| 2a75f40871 | |||
| a705ba74a1 | |||
| 8d9c9e7c87 | |||
| 0b5cc4f251 | |||
| 82ffe85f2d | |||
| b3644abc0e | |||
| d653bf1b90 | |||
| 3e66127b94 | |||
| 5d086c13ca | |||
| d36d4e7689 | |||
| 1456aa51cc | |||
| ffc179c822 | |||
| 8e669a1f14 | |||
| 14bfae216d | |||
| 5a58d87995 | |||
| 557809f364 | |||
| ee600975f0 | |||
| 0d8017e23f | |||
| 37b18edf3d | |||
| cda7a61b47 | |||
| 2ad9283148 | |||
| 41a8a80dda | |||
| 90c7fa6381 | |||
| e3839f8620 | |||
| 0fc2f9ee01 | |||
| 5d6305f21a | |||
| 551e4daea2 | |||
| 3589d33468 | |||
| a22c1a7f59 | |||
| ad85d71b0f | |||
| f3506240c2 | |||
| 70c199aa31 | |||
| e2b82746ab | |||
| 4235507f7b | |||
| b346670cc7 | |||
| 7868c26be7 |
33
.woodpecker/release.yml
Normal file
33
.woodpecker/release.yml
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
when:
|
||||||
|
- event: tag
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: validate-release
|
||||||
|
image: golang:1.25.5
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
version="$CI_COMMIT_TAG"
|
||||||
|
release_note="docs/releases/$version.md"
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
|
||||||
|
printf '%s\n' "invalid release tag: $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$release_note" ]; then
|
||||||
|
printf '%s\n' "missing release note: $release_note" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note heading does not match $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note is missing heading: $heading" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
./scripts/check-release-source.sh "$version"
|
||||||
16
README.md
16
README.md
@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
|
|||||||
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
||||||
demonstrates all implemented D&D lanes and the supporting campaign references.
|
demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||||
|
|
||||||
|
## Install A Source Release
|
||||||
|
|
||||||
|
Install a pinned source release with Go:
|
||||||
|
|
||||||
|
~~~
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
|
||||||
|
installed command's diagnostic version is described in the [CLI
|
||||||
|
reference](docs/cli.md); maintainers preparing a release should follow [Source
|
||||||
|
Releases](docs/release.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
||||||
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
|||||||
artifact formats.
|
artifact formats.
|
||||||
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
||||||
from an orchestrator and consume a published result.
|
from an orchestrator and consume a published result.
|
||||||
|
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
|
||||||
|
D&D pipeline as a subprocess and discover its structured artifacts.
|
||||||
- [Internal overview](docs/internal/overview.md) — implemented component map
|
- [Internal overview](docs/internal/overview.md) — implemented component map
|
||||||
for maintainers.
|
for maintainers.
|
||||||
- [Developer guide](docs/development.md) — contributor orientation and
|
- [Developer guide](docs/development.md) — contributor orientation and
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_combat_turns_llm.v1.json
|
schema_path: dnd_combat_turns_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -50,4 +50,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_enemy_events_llm.v1.json
|
schema_path: dnd_enemy_events_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
||||||
"$id": "notarius.dnd.entity_reconcile.llm",
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["duplicate_groups"],
|
|
||||||
"properties": {
|
|
||||||
"duplicate_groups": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["members", "canonical"],
|
|
||||||
"properties": {
|
|
||||||
"members": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"$ref": "#/$defs/selector"}
|
|
||||||
},
|
|
||||||
"canonical": {"$ref": "#/$defs/selector"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"$defs": {
|
|
||||||
"selector": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["name", "source_refs"],
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string", "minLength": 1},
|
|
||||||
"source_refs": {
|
|
||||||
"type": "array",
|
|
||||||
"minItems": 1,
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["start_unit_id", "end_unit_id"],
|
|
||||||
"properties": {
|
|
||||||
"start_unit_id": {"type": "integer", "minimum": 1},
|
|
||||||
"end_unit_id": {"type": "integer", "minimum": 1}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_occurrences_llm.v1.json
|
schema_path: dnd_item_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -22,10 +22,10 @@
|
|||||||
"items": {
|
"items": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"required": ["start_segment", "end_segment"],
|
"required": ["start_unit_id", "end_unit_id"],
|
||||||
"properties": {
|
"properties": {
|
||||||
"start_segment": {"type": "integer"},
|
"start_unit_id": {"type": "integer"},
|
||||||
"end_segment": {"type": "integer"}
|
"end_unit_id": {"type": "integer"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_registry_llm.v1.json
|
schema_path: dnd_item_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
Use candidate names and cited transcript windows only to determine whether
|
Determine whether candidates identify the same item type or unique designation
|
||||||
candidates identify the same item type or unique designation. Do not treat
|
using their contextual labels and cited transcript windows. Do not treat nearby
|
||||||
nearby evidence, similar objects, or a shared owner as sufficient. Keep
|
evidence, similar objects, or a shared owner as sufficient.
|
||||||
currency denominations, materially different item types, and uncertain aliases
|
|
||||||
separate. Do not infer an item property or uniqueness.
|
Keep currency denominations and materially different item types separate. Keep
|
||||||
|
uncertain aliases separate. Do not infer an item property or uniqueness.
|
||||||
|
|
||||||
When selecting a canonical display name, choose one supplied candidate name
|
When selecting a canonical display name, choose one supplied candidate name
|
||||||
that is the clearest established designation.
|
that is the clearest established designation.
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_occurrences_llm.v1.json
|
schema_path: dnd_location_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_registry_llm.v1.json
|
schema_path: dnd_location_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
Location candidates:
|
|
||||||
{{ input "candidates" }}
|
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
Use candidate names and their cited transcript windows to determine whether
|
Determine whether candidates identify the same physical place using their
|
||||||
candidates identify the same physical place. Do not treat matching names,
|
contextual labels and cited transcript windows. Do not treat matching names,
|
||||||
nearby evidence, nested places, or generic labels as sufficient. Keep parent
|
nearby evidence, nested places, or generic labels as sufficient.
|
||||||
and child places, similarly named places, and uncertain aliases separate.
|
|
||||||
|
Keep parent and child places separate, as well as similarly named places and
|
||||||
|
uncertain aliases.
|
||||||
|
|
||||||
When selecting a canonical display name, prefer the clearest established name.
|
When selecting a canonical display name, prefer the clearest established name.
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_registry_llm.v1.json
|
schema_path: dnd_npc_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
NPC candidates for identity comparison:
|
|
||||||
|
|
||||||
{{ input "candidates" }}
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
Use candidate aliases and their cited transcript windows to determine whether
|
Determine whether candidates refer to the same individual using their
|
||||||
candidates refer to the same individual. Preserve distinct individuals even
|
contextual labels and cited transcript windows. Preserve distinct individuals
|
||||||
when their names are similar.
|
even when their names are similar or their contextual descriptions are
|
||||||
|
identical.
|
||||||
|
|
||||||
When selecting a canonical display name, prefer a complete, stable proper name
|
When selecting a canonical display name, prefer a complete, stable proper name
|
||||||
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -35,4 +35,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -31,4 +31,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scenes_llm.v1.json
|
schema_path: dnd_scenes_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
Identify only well-supported duplicate groups among the supplied candidates.
|
|
||||||
|
|
||||||
Return each selected candidate's supplied contextual descriptor exactly: its
|
|
||||||
`name` and complete ordered `source_refs`. A group must contain at least two
|
|
||||||
supplied descriptors, and its `canonical` descriptor must be one of its
|
|
||||||
members. Do not invent names, ranges, records, evidence, or replacement values.
|
|
||||||
Omit any uncertain or unsafe group.
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
Transcript units are the only evidence for extracted events and factual claims.
|
Transcript units are the only evidence for extracted events and factual claims.
|
||||||
Every reported factual claim must be supported by cited transcript units. Use
|
Every reported factual claim must be supported by cited transcript units. Use
|
||||||
integer `start_unit_id` and `end_unit_id` values from the transcript. Omit
|
integer `start_unit_id` and `end_unit_id` values from the transcript.
|
||||||
`source_id`; Notarius assigns the current source identity.
|
|
||||||
|
|
||||||
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
||||||
than a broad range that bridges unrelated conversation.
|
than a broad range that bridges unrelated conversation.
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
You process Dungeons & Dragons gameplay transcripts.
|
You process Dungeons & Dragons gameplay transcripts.
|
||||||
|
|
||||||
Rely only on the supplied inputs. They may contain transcription errors,
|
As input, you will receive one or more portions of a transcript. The transcript may contain transcription errors, repeated lines, incomplete sentences, and misheard proper nouns.
|
||||||
repeated lines, incomplete sentences, and misheard proper nouns.
|
|
||||||
|
|
||||||
Return exactly one JSON object that conforms to the configured response schema,
|
Return exactly one JSON object that conforms to the configured response schema, with no explanatory prose.
|
||||||
with no explanatory prose.
|
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
One extraction chunk from a Dungeons & Dragons gameplay transcript is provided
|
One extraction chunk from a Dungeons & Dragons gameplay transcript is provided below. Report and infer only what is within this chunk. Its unit IDs retain their source-wide meaning.
|
||||||
below. Report and infer only what is within this chunk. Its unit IDs retain
|
|
||||||
their source-wide meaning.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
{{ input "transcript" }}
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
The complete ordered transcript of this Dungeons & Dragons gameplay session is
|
The complete ordered transcript of this Dungeons & Dragons gameplay session is provided below.
|
||||||
provided below. It may contain multiple scenes.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
{{ input "transcript" }}
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
Selected Dungeons & Dragons gameplay transcript evidence windows are provided
|
|
||||||
below. They may be incomplete, non-contiguous, or overlapping. Use them to
|
|
||||||
evaluate candidate identity, but do not treat absence outside these windows as
|
|
||||||
evidence.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
|
||||||
@@ -47,4 +47,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_spells_llm.v1.json
|
schema_path: dnd_spells_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
The canonical spell-name catalog for this extraction is provided below as JSON.
|
The spell catalog for this extraction is provided below as JSON. Each entry
|
||||||
Return spell names using the catalog's canonical spelling exactly. Aliases and
|
lists a `canonical_name` and its recognized `aliases`. If the transcript uses
|
||||||
other campaign reference material are not part of this catalog input and must
|
an alias, select that entry's `canonical_name`. Return spell names using the
|
||||||
not be copied into the output as spell names.
|
canonical spelling exactly; never return an alias as a spell name.
|
||||||
|
|
||||||
{{ input "spell_catalog" }}
|
{{ input "spell_catalog" }}
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
Item candidates:
|
Candidate material:
|
||||||
|
|
||||||
{{ input "candidates" }}
|
{{ input "candidates" }}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
Identify only high-confidence duplicate entities among the supplied candidates.
|
||||||
|
|
||||||
|
Preserve distinct entities even when their names are similar. Treat contextual descriptions and transcript evidence as supporting material, not as permission to merge ambiguous records.
|
||||||
|
|
||||||
|
When several records are duplicates, choose as canonical the candidate with the clearest stable identity. Prefer a complete proper name over an abbreviation, and prefer an unadorned proper name over one with incidental descriptors unless the evidence establishes those descriptors as part of the name. A longer name is not inherently more canonical.
|
||||||
27
assets/generic/normalize/deduplication/prompts/prompt.yaml
Normal file
27
assets/generic/normalize/deduplication/prompts/prompt.yaml
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
id: generic.semantic_reconciliation
|
||||||
|
version: "v1"
|
||||||
|
inputs:
|
||||||
|
- name: candidates
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content_file: ./system.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./protocol.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./instructions.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./candidates.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./transcript-windows.md
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Use only the positive integer `candidate_id` values supplied in the candidate material.
|
||||||
|
|
||||||
|
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group.
|
||||||
|
|
||||||
|
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid.
|
||||||
|
|
||||||
|
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||||
2
assets/generic/normalize/deduplication/prompts/system.md
Normal file
2
assets/generic/normalize/deduplication/prompts/system.md
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
You reconcile structured records that may describe the same underlying entity.
|
||||||
|
Follow the supplied protocol and return only the requested structured result.
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
Transcript evidence windows:
|
||||||
|
|
||||||
|
{{ input "transcript" }}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "notarius.generic.semantic_reconciliation.llm",
|
||||||
|
"title": "notarius_semantic_reconciliation_llm_v1",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["duplicate_groups"],
|
||||||
|
"properties": {
|
||||||
|
"duplicate_groups": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["candidate_ids", "canonical_candidate_id"],
|
||||||
|
"properties": {
|
||||||
|
"candidate_ids": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 2,
|
||||||
|
"items": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"canonical_candidate_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,9 +12,9 @@ an opaque implementation detail. A plain name is likewise insufficient where
|
|||||||
multiple supplied records share that name.
|
multiple supplied records share that name.
|
||||||
|
|
||||||
The LLM boundary must preserve the typed artifact and durable-schema ownership
|
The LLM boundary must preserve the typed artifact and durable-schema ownership
|
||||||
of [ADR-0003](0003-strongly-typed-stage-interfaces.md) and the distinction
|
of [ADR-0003](0003-typed-interfaces-with-two-zone-data-model.md) and the distinction
|
||||||
between disambiguating references and source evidence in
|
between disambiguating references and source evidence in
|
||||||
[ADR-0009](0009-prefer-minimal-evidence-grounded-extraction-artifacts.md).
|
[ADR-0009](0009-minimal-evidence-grounded-extraction-artifacts.md).
|
||||||
|
|
||||||
## Decision
|
## Decision
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
# ADR-0013: Use request-local candidate handles for semantic reconciliation
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-08-09
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Several typed normalize stage modules need semantic reconciliation after
|
||||||
|
deterministic preprocessing: a model can judge whether source-backed candidates
|
||||||
|
refer to the same underlying entity, while application code remains responsible
|
||||||
|
for constructing the normalized artifact. Requiring the model to reproduce a
|
||||||
|
candidate's full contextual selector makes the response larger and introduces
|
||||||
|
avoidable formatting, ordering, and transcription failure modes.
|
||||||
|
|
||||||
|
Reconciliation must preserve the exact typed artifact boundary established by
|
||||||
|
[ADR-0003](0003-typed-interfaces-with-two-zone-data-model.md), the domain-neutral
|
||||||
|
framework and concrete-domain dependency direction established by
|
||||||
|
[ADR-0004](0004-package-modules-by-domain.md), and the distinction in
|
||||||
|
[ADR-0009](0009-minimal-evidence-grounded-extraction-artifacts.md) between source
|
||||||
|
evidence and auxiliary identity context. It also needs a concrete, narrowly
|
||||||
|
scoped application of the request-local-label exception allowed by
|
||||||
|
[ADR-0012](0012-resolve-opaque-entity-identifiers-deterministically.md).
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Semantic reconciliation will be a domain-neutral framework mechanism used by
|
||||||
|
typed normalize stage modules. A consuming artifact family will retain
|
||||||
|
ownership of its typed records, identity rules, consolidation policy, durable
|
||||||
|
IDs, and domain warnings; the framework mechanism will not infer those rules
|
||||||
|
from arbitrary data.
|
||||||
|
|
||||||
|
For each reconciliation request, deterministic code will assign every eligible
|
||||||
|
model-visible candidate a contiguous, one-based integer handle. The model may
|
||||||
|
receive the candidate's contextual label, source references, and bounded source
|
||||||
|
context needed to judge identity, but its structured response will identify
|
||||||
|
candidates only by those supplied handles. A handle is local to one request,
|
||||||
|
does not represent entity identity, and must never enter a durable artifact or
|
||||||
|
be used to derive a durable ID.
|
||||||
|
|
||||||
|
The model will propose duplicate groups and select one supplied member of each
|
||||||
|
group as canonical. Deterministic code will resolve the handles through the
|
||||||
|
retained request mapping, validate the complete proposal, discard unsafe
|
||||||
|
groups, and apply only validated groups through typed domain-owned policy. The
|
||||||
|
model will not synthesize replacement records or directly mutate an artifact.
|
||||||
|
|
||||||
|
Every reconciliation prompt will combine a mandatory framework-owned protocol
|
||||||
|
and safety policy with an explicitly selected semantic policy. The semantic
|
||||||
|
policy may be the conservative generic policy or a domain-owned policy, but it
|
||||||
|
cannot replace the shared response protocol or deterministic safety boundary.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Return durable application IDs. Opaque IDs do not help semantic judgment,
|
||||||
|
expose application identity mechanics, and make model output reproduce data
|
||||||
|
that deterministic code already owns.
|
||||||
|
- Return names alone or copied contextual selectors. Names can be ambiguous,
|
||||||
|
while reproducing labels and source ranges adds response complexity and
|
||||||
|
creates mismatches without adding semantic information. Request-local
|
||||||
|
handles preserve exact selection without either failure mode.
|
||||||
|
- Ask the model to return synthesized canonical replacement records. This
|
||||||
|
would transfer typed artifact construction, provenance consolidation, and
|
||||||
|
durable identity policy to a probabilistic boundary.
|
||||||
|
- Reconcile reflection-discovered fields or arbitrary JSON. This would weaken
|
||||||
|
the typed artifact contract and move domain semantics into generic code.
|
||||||
|
- Hide reconciliation inside extraction or another stage. This would obscure
|
||||||
|
stage ownership and create cross-stage behavior outside the fixed pipeline;
|
||||||
|
reconciliation remains explicit normalize-stage behavior.
|
||||||
|
- Let each domain replace the complete prompt protocol. This would duplicate
|
||||||
|
safety mechanics and allow domain policy to bypass the common response and
|
||||||
|
validation contract.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Model responses become smaller and easier to validate, while deterministic
|
||||||
|
application code retains authority over identity, provenance, ordering, and
|
||||||
|
typed artifact construction. The framework requires a request-local mapping,
|
||||||
|
bounded context preparation, a private integer response contract, proposal
|
||||||
|
assessment, and shared prompt assets. Each consuming artifact family still
|
||||||
|
requires a typed adapter for its irreducibly domain-specific rules.
|
||||||
|
|
||||||
|
Request-local handles are deliberately unsuitable for persistence, logging as
|
||||||
|
entity identity, checkpoint contracts, or cross-request correlation. Changes
|
||||||
|
to shared protocol and policy assets must participate in the normal prompt,
|
||||||
|
schema, and checkpoint fingerprint mechanisms.
|
||||||
|
|
||||||
|
The shared mechanism and its initial D&D registry consumers are now
|
||||||
|
implemented. Current behavior is documented in
|
||||||
|
[Module Internals](../internal/modules.md#semantic-reconciliation) and
|
||||||
|
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||||
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# ADR-0014: Use feedback-aware validation retries
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-08-26
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Validation can identify a candidate defect after a producer has returned an
|
||||||
|
otherwise well-formed result. Retrying without the validator's deterministic,
|
||||||
|
bounded feedback wastes the useful diagnosis, while treating validator
|
||||||
|
execution failures as defects would ask a producer to repair conditions it
|
||||||
|
cannot control. The mechanism must preserve typed producer ownership,
|
||||||
|
checkpoint safety, and the repository's sensitive-data boundaries.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
The implementation will keep three independent budgets: the producer binding's
|
||||||
|
outer `retries` budget, PromptKit's structured-output repair budget, and each
|
||||||
|
validator's execution-retry budget. Validators will run sequentially in their
|
||||||
|
configured order and aggregate both rejections and execution failures before a
|
||||||
|
candidate disposition is selected.
|
||||||
|
|
||||||
|
A correction-capable producer will provide the exact single LLM response that
|
||||||
|
controlled its candidate using the `single_response_v1` protocol. A correction
|
||||||
|
attempt will reconstruct the ordinary request and append exactly two fresh
|
||||||
|
messages: that latest response as `assistant`, followed by one deterministic
|
||||||
|
aggregate correction request as `user`. Earlier turns will not accumulate.
|
||||||
|
|
||||||
|
Validator failures will not recurse into correction. Pipeline policy owns
|
||||||
|
terminal disposition, with field-by-field producer overrides over pipeline
|
||||||
|
defaults: structural failure and semantic rejection default to `fail_run`, and
|
||||||
|
validator execution failure defaults to `warn_continue`. Validators can report
|
||||||
|
facts and bounded corrective guidance, but never decide disposition.
|
||||||
|
|
||||||
|
Rejected and structurally invalid candidates will not advance. A candidate
|
||||||
|
allowed through after a validator execution failure will retain explicit
|
||||||
|
incomplete-validation provenance and will not be checkpointed. Exact response
|
||||||
|
and correction text remain attempt-local: they are excluded from ordinary
|
||||||
|
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
|
||||||
|
summaries.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Retry every producer after any validation outcome. This conflates producer
|
||||||
|
defects with validator operational failures and wastes retry budget.
|
||||||
|
- Let validators decide whether to continue. This would distribute pipeline
|
||||||
|
disposition policy across validators and undermine consistent defaults.
|
||||||
|
- Reuse the full prior conversation. Accumulated turns introduce unbounded
|
||||||
|
prompt growth and make correction behavior depend on incidental history.
|
||||||
|
- Persist raw responses to simplify diagnosis. Raw model output and correction
|
||||||
|
guidance may be sensitive and do not belong in durable pipeline records.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
The framework gains transport-neutral correction and candidate contracts,
|
||||||
|
producer capability checks, policy resolution, aggregated validation outcomes,
|
||||||
|
and conservative checkpoint handling. Prompt construction remains inside the
|
||||||
|
LLM adapter, while modules remain responsible for accurately exposing the
|
||||||
|
single response that directly controlled a candidate.
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# ADR-0015: Separate process warnings from quality diagnostics
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-08-27
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Notarius currently represents process degradation, incomplete validation,
|
||||||
|
extraction-quality doubt, and routine normalization with one flat warning
|
||||||
|
record. That makes ordinary successful runs noisy, loses the framework context
|
||||||
|
needed to explain a finding, and gives `warning_count` no stable operational
|
||||||
|
meaning. It also permits output encoders to add a warning after the durable
|
||||||
|
warning file has already been written.
|
||||||
|
|
||||||
|
The application needs one bounded diagnostic model that preserves exact
|
||||||
|
occurrence counts while retaining only safe, representative samples. Fresh and
|
||||||
|
resumed logical runs must present the same groups. The model must not alter
|
||||||
|
validation decisions, retry budgets, rejected-output behavior, or process exit
|
||||||
|
policy.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Warnings are reserved for a completed run that advanced under an allowed
|
||||||
|
process-level degradation or incomplete-work policy. Extraction-quality signals
|
||||||
|
are advisories, and routine accepted transformations are observations. A
|
||||||
|
non-degraded successful run therefore has zero actionable warnings.
|
||||||
|
|
||||||
|
Modules and validators own a diagnostic's disposition, category, reason code,
|
||||||
|
scope, and safe message. The framework adds pipeline origin, including stage,
|
||||||
|
step, lane, module, validator, and chunk context where applicable. It then
|
||||||
|
aggregates deterministically by disposition, category, reason code, and full
|
||||||
|
origin. Chunk context remains on representative samples so equivalent findings
|
||||||
|
across chunks aggregate together.
|
||||||
|
|
||||||
|
Diagnostics carry exact occurrence counts, at most three distinct samples, and
|
||||||
|
numeric omitted-sample metadata. Producers and validators are bounded to 64
|
||||||
|
local groups. Final actionable warning groups are bounded without truncation;
|
||||||
|
the non-warning collection may truncate represented groups while preserving an
|
||||||
|
exact total occurrence count and explicit truncation metadata.
|
||||||
|
|
||||||
|
The public contracts will be versioned: grouped actionable warnings use
|
||||||
|
`notarius.warnings.v2`, grouped advisories and observations use
|
||||||
|
`notarius.diagnostics.v1`, and the run receipt uses
|
||||||
|
`notarius.run-result.v2`. Successful output encoders return logical files or
|
||||||
|
an error; they do not add post-encoding warnings.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Keep one warning list and filter only CLI output. This would leave durable
|
||||||
|
consumers with the same semantically mixed, unbounded contract.
|
||||||
|
- Map reason codes to severity in a central framework registry. This would
|
||||||
|
split module-owned meaning between synchronized policy tables and make new
|
||||||
|
diagnostic meaning implicit.
|
||||||
|
- Preserve local omission warning records. They inflate visible group counts
|
||||||
|
and lose exact occurrence semantics.
|
||||||
|
- Keep output-encoder warnings. A one-pass encoder cannot include those
|
||||||
|
records consistently in files it has already serialized; a two-phase encoder
|
||||||
|
protocol is deferred until a demonstrated need exists.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
The framework gains validated diagnostic primitives, local collection,
|
||||||
|
origin-aware aggregation, and versioned durable presentation. Existing warning
|
||||||
|
transport remains temporarily while producers migrate. Current architecture,
|
||||||
|
operator, integration, and internal documentation will describe the behavior
|
||||||
|
only as each implementation step lands; this accepted decision does not claim
|
||||||
|
that the migration is complete.
|
||||||
21
docs/cli.md
21
docs/cli.md
@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
|
|||||||
|
|
||||||
~~~
|
~~~
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||||
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
|
|||||||
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
||||||
writes the command summary to standard output and exits with status 0.
|
writes the command summary to standard output and exits with status 0.
|
||||||
|
|
||||||
|
`notarius --version` is valid only as the sole root argument. It writes exactly
|
||||||
|
`notarius <version>` followed by a newline to standard output and exits with
|
||||||
|
status 0. A tagged `go install` build can report its main-module stable tag,
|
||||||
|
and controlled builds can inject a stable tag at link time through
|
||||||
|
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
|
||||||
|
unversioned checkout reports `development`. Invalid injected version content is
|
||||||
|
a runtime error with exit status 1, while extra `--version` arguments are a
|
||||||
|
syntax error with exit status 2. This diagnostic does not replace the
|
||||||
|
[run-result](integrations/run-result.md) or artifact contracts for downstream
|
||||||
|
compatibility decisions.
|
||||||
|
|
||||||
## run
|
## run
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
@@ -91,11 +103,14 @@ names, requiredness, and configured bindings are part of the
|
|||||||
Without **--json**, standard output contains the completed pipeline ID, counts
|
Without **--json**, standard output contains the completed pipeline ID, counts
|
||||||
of normalized and rejected outputs, and the output directory. A debug-enabled
|
of normalized and rejected outputs, and the output directory. A debug-enabled
|
||||||
run also prints its debug-bundle path to standard output. A successful run with
|
run also prints its debug-bundle path to standard output. A successful run with
|
||||||
warnings reports the warning count to standard error. The published JSON bundle
|
actionable process warnings reports their group and occurrence counts to
|
||||||
|
standard error. When the selected output module publishes `warnings.json`, the
|
||||||
|
summary also reports that durable file's path. Advisory and observation findings
|
||||||
|
do not produce a warning line. The published JSON bundle
|
||||||
is defined by the [JSON output contract](integrations/json-output.md).
|
is defined by the [JSON output contract](integrations/json-output.md).
|
||||||
|
|
||||||
With **--json**, successful standard output is exactly one
|
With **--json**, successful standard output is exactly one
|
||||||
`notarius.run-result.v1` JSON document followed by a newline, with no
|
`notarius.run-result.v2` JSON document followed by a newline, with no
|
||||||
human-oriented status or debug-path line. Its fields and compatibility policy
|
human-oriented status or debug-path line. Its fields and compatibility policy
|
||||||
are defined by the [run-result contract](integrations/run-result.md). A caller
|
are defined by the [run-result contract](integrations/run-result.md). A caller
|
||||||
must check for exit status 0 before decoding this output; a failed write can
|
must check for exit status 0 before decoding this output; a failed write can
|
||||||
@@ -159,7 +174,7 @@ go run ./cmd/notarius pipelines list \
|
|||||||
Successful commands write their primary result to standard output. Warnings and
|
Successful commands write their primary result to standard output. Warnings and
|
||||||
errors are written to standard error.
|
errors are written to standard error.
|
||||||
|
|
||||||
For **run --json**, warnings remain on standard error and standard output is a
|
For **run --json**, actionable process warnings remain on standard error and standard output is a
|
||||||
machine-readable success result only. Syntax and runtime diagnostics remain on
|
machine-readable success result only. Syntax and runtime diagnostics remain on
|
||||||
standard error. Parse the result only after the process exits with status 0.
|
standard error. Parse the result only after the process exits with status 0.
|
||||||
|
|
||||||
|
|||||||
@@ -132,7 +132,11 @@ model: example-model
|
|||||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||||
its credential environment variable through `api_key_env`; set that variable
|
its credential environment variable through `api_key_env`; set that variable
|
||||||
only in the run environment. PromptKit owns the
|
only in the run environment. PromptKit owns the
|
||||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
|
||||||
|
including `base_profile` inheritance. Notarius passes profiles through without
|
||||||
|
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||||
|
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||||
|
provider without authorization.
|
||||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||||
supported package API, and [Operations](operations.md#operational-limits)
|
supported package API, and [Operations](operations.md#operational-limits)
|
||||||
describes the effective concurrency layers.
|
describes the effective concurrency layers.
|
||||||
@@ -218,6 +222,8 @@ pipelines:
|
|||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||||
|
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
|
||||||
|
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
|
||||||
| **input** | module binding | none | Required. |
|
| **input** | module binding | none | Required. |
|
||||||
| **chunk** | module binding | **generic** | Optional. |
|
| **chunk** | module binding | **generic** | Optional. |
|
||||||
| **output** | module binding | **json** | Optional. |
|
| **output** | module binding | **json** | Optional. |
|
||||||
@@ -237,6 +243,42 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
|
|||||||
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
||||||
Deterministic bindings do not receive these defaults or run overrides.
|
Deterministic bindings do not receive these defaults or run overrides.
|
||||||
|
|
||||||
|
Structural output repair is resolved after module, validator, and `--only` lane
|
||||||
|
selection. An object's **structured_output_repair_attempts** value takes
|
||||||
|
precedence over the pipeline value; otherwise, an LLM-backed binding or
|
||||||
|
validator inherits the pipeline value. If both are omitted, PromptKit uses the
|
||||||
|
prompt's declared repair policy. The value must be an integer from 0 through 3;
|
||||||
|
explicit `null` and non-integer values are invalid. An explicit value on a
|
||||||
|
deterministic binding or validator is invalid, while a pipeline value simply
|
||||||
|
does not apply to deterministic selections.
|
||||||
|
|
||||||
|
`validation_policy` controls terminal disposition for one complete producer
|
||||||
|
attempt and validator chain. It may appear on a pipeline or a **chunk**,
|
||||||
|
**extract**, **merge**, or **normalize** module binding; input, output, and
|
||||||
|
validator bindings reject it. Every field is optional and resolves in binding,
|
||||||
|
pipeline, then application-default order:
|
||||||
|
|
||||||
|
| Field | Values | Default |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
|
||||||
|
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
|
||||||
|
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
|
||||||
|
|
||||||
|
The policy object and its fields must be non-null, and unknown fields are
|
||||||
|
rejected. A deterministic producer may not explicitly set
|
||||||
|
**producer_structural_failure** on its binding, although a pipeline-level
|
||||||
|
default remains valid for pipelines that include LLM-backed producers.
|
||||||
|
|
||||||
|
After the producer binding's retry budget is exhausted, an invalid structured
|
||||||
|
response uses **producer_structural_failure**. One or more semantic validator
|
||||||
|
rejections use **semantic_rejection**; rejection takes precedence over an
|
||||||
|
exhausted validator failure or skip. With no rejection, an exhausted validator
|
||||||
|
failure or skip uses **validator_failure**. `reject_output` records the
|
||||||
|
terminal rejection without advancing that candidate. `warn_continue` is valid
|
||||||
|
only for validator execution failure: it advances a structurally valid,
|
||||||
|
otherwise unrejected result with incomplete-validation provenance and without
|
||||||
|
making it reusable checkpoint state.
|
||||||
|
|
||||||
A lane has these fields:
|
A lane has these fields:
|
||||||
|
|
||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
@@ -274,17 +316,32 @@ extract:
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
|
||||||
|
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
|
||||||
|
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
|
||||||
| **options** | object | none | Must satisfy the selected module. |
|
| **options** | object | none | Must satisfy the selected module. |
|
||||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||||
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||||
|
|
||||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||||
Validator bindings accept only **module**, **llm_profile**, and **options**.
|
Validator bindings accept only **module**, **llm_profile**,
|
||||||
They reject **references**, **retries**, and nested **validators**. Deterministic
|
**structured_output_repair_attempts**, **retries**, and **options**. Their
|
||||||
validators reject an explicit **llm_profile**. Deterministic module bindings
|
**retries** value is a non-negative additional validator-execution budget and
|
||||||
also reject an explicit **llm_profile**.
|
is valid only when the selected validator is LLM-backed. A validator retry
|
||||||
|
rechecks the same immutable candidate; it never regenerates the producer.
|
||||||
|
They reject
|
||||||
|
**validation_policy**, **references**, and nested **validators**. Deterministic
|
||||||
|
validators reject explicit **llm_profile** and
|
||||||
|
**structured_output_repair_attempts**.
|
||||||
|
Deterministic module bindings also reject those explicit fields.
|
||||||
|
|
||||||
|
An LLM-backed chunk, extract, merge, or normalize producer with both a
|
||||||
|
non-empty validator chain and positive **retries** must declare the supported
|
||||||
|
single-response correction capability. Preparation rejects a configuration
|
||||||
|
that could require semantic correction from a producer that cannot provide an
|
||||||
|
exact prior response. A deterministic producer, or an LLM attempt that did
|
||||||
|
not make a model call, cannot consume a semantic retry after rejection.
|
||||||
|
|
||||||
The **json** output module accepts optional **include_chunk_map** and
|
The **json** output module accepts optional **include_chunk_map** and
|
||||||
**evidence_context** settings:
|
**evidence_context** settings:
|
||||||
@@ -319,7 +376,8 @@ Unknown outer or nested option fields are rejected, as are incompatible YAML
|
|||||||
types. The allowlist remains valid when a run uses lane filtering: a configured
|
types. The allowlist remains valid when a run uses lane filtering: a configured
|
||||||
lane that is not active for that invocation simply contributes no evidence.
|
lane that is not active for that invocation simply contributes no evidence.
|
||||||
Evidence publication is opt-in because it can persist source text and metadata.
|
Evidence publication is opt-in because it can persist source text and metadata.
|
||||||
Its payload contract is [Published Evidence Context](integrations/evidence-context.md).
|
When enabled, it publishes the selected source-unit excerpt defined by the
|
||||||
|
[Published Evidence Context contract](integrations/evidence-context.md).
|
||||||
|
|
||||||
## References And Ordered Handoffs
|
## References And Ordered Handoffs
|
||||||
|
|
||||||
|
|||||||
209
docs/consumers/dnd-pipeline.md
Normal file
209
docs/consumers/dnd-pipeline.md
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
# Consuming The Complete D&D Pipeline
|
||||||
|
|
||||||
|
Use this workflow when an orchestrator runs the maintained complete D&D
|
||||||
|
pipeline and consumes its structured JSON artifacts. The generic
|
||||||
|
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
|
||||||
|
this guide connects that workflow to the complete D&D configuration, its
|
||||||
|
Seriatim input, and its artifact inventory.
|
||||||
|
|
||||||
|
The [CLI reference](../cli.md), [configuration reference](../config.md),
|
||||||
|
[run-result receipt](../integrations/run-result.md), and
|
||||||
|
[published JSON output contract](../integrations/json-output.md) remain the
|
||||||
|
canonical definitions of those public interfaces.
|
||||||
|
|
||||||
|
## Prepare And Validate The Deployment
|
||||||
|
|
||||||
|
Start from the maintained
|
||||||
|
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
|
||||||
|
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
|
||||||
|
artifact handoffs, campaign references, chunk-map publication, and evidence
|
||||||
|
context.
|
||||||
|
|
||||||
|
A deployment must provide its own PromptKit profile and campaign reference
|
||||||
|
files. Use absolute paths for service and subprocess deployments. In
|
||||||
|
particular, observe these different resolution rules:
|
||||||
|
|
||||||
|
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||||
|
file; and
|
||||||
|
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||||
|
directory.
|
||||||
|
|
||||||
|
Do not copy the repository example's relative profile path into a deployment
|
||||||
|
without also controlling that working directory. The complete path and profile
|
||||||
|
rules are defined in [Configuration](../config.md).
|
||||||
|
|
||||||
|
Preflight the deployed configuration before processing sessions and whenever
|
||||||
|
it changes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius config validate \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--pipeline dnd-session
|
||||||
|
```
|
||||||
|
|
||||||
|
Provide credentials through the environment or the documented configuration
|
||||||
|
mechanism. Do not put credentials in command arguments, generated
|
||||||
|
configuration, or logs.
|
||||||
|
|
||||||
|
## Supply The Transcript
|
||||||
|
|
||||||
|
The complete pipeline consumes a Seriatim JSON document. The
|
||||||
|
[Seriatim input contract](../integrations/seriatim.md) defines its required
|
||||||
|
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
|
||||||
|
citations use those segment IDs as source-unit ranges.
|
||||||
|
|
||||||
|
When the caller maintains several transcript tiers, use the final trimmed JSON
|
||||||
|
transcript so extraction operates on the same session content presented to
|
||||||
|
later consumers. For example, Narratio identifies this implemented artifact as
|
||||||
|
`narratio.transcript.final_trimmed` and normally stores it at
|
||||||
|
`transcripts/final.trimmed.json`.
|
||||||
|
|
||||||
|
Notarius generates a stable prompt session from the resolved input module and
|
||||||
|
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
|
||||||
|
Use that override only when intentionally changing the routing relationship
|
||||||
|
between invocations; it is not a credential or output identity.
|
||||||
|
|
||||||
|
## Run Notarius
|
||||||
|
|
||||||
|
Invoke the pipeline with explicit absolute paths and request its
|
||||||
|
machine-readable receipt:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius run dnd-session \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||||
|
--output-dir /absolute/path/to/notarius-output \
|
||||||
|
--json
|
||||||
|
```
|
||||||
|
|
||||||
|
The caller should:
|
||||||
|
|
||||||
|
- capture stdout and stderr separately;
|
||||||
|
- propagate cancellation and impose an operator-appropriate timeout;
|
||||||
|
- wait for process completion before interpreting stdout; and
|
||||||
|
- retain stderr for diagnosis without copying secrets or transcript content
|
||||||
|
into other logs.
|
||||||
|
|
||||||
|
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
|
||||||
|
nonzero exit because a failed receipt write can leave partial bytes. The
|
||||||
|
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
|
||||||
|
stream and exit-status contract.
|
||||||
|
|
||||||
|
## Discover The Published Bundle
|
||||||
|
|
||||||
|
Decode the successful stdout document as a supported run-result schema. For
|
||||||
|
the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate
|
||||||
|
unknown fields allowed by that version, but reject an unsupported schema
|
||||||
|
version.
|
||||||
|
|
||||||
|
Use the receipt's absolute `output_directory` as the exact run-specific bundle
|
||||||
|
root. Do not scan the output root for its newest directory, guess a run ID, or
|
||||||
|
construct a bundle path. Resolve `index_file` beneath `output_directory` and
|
||||||
|
reject an absolute logical path or any result that escapes the bundle root.
|
||||||
|
|
||||||
|
The complete configuration uses the application validation defaults. A caller
|
||||||
|
that requires fully validated D&D artifacts must also require receipt
|
||||||
|
`validation_status: approved`; a successful `incomplete` result reflects the
|
||||||
|
configured validator-failure continuation policy and carries its bounded
|
||||||
|
validator provenance in `validation_summaries`.
|
||||||
|
|
||||||
|
Read `index.json` and locate each requested lane in `output_files` by its exact
|
||||||
|
`lane_id`. Do not guess a lane filename. Before decoding a payload:
|
||||||
|
|
||||||
|
1. resolve its descriptor's relative `file` beneath the bundle root with the
|
||||||
|
same confinement check;
|
||||||
|
2. verify the descriptor's media type and schema identity against the linked
|
||||||
|
artifact contract; and
|
||||||
|
3. decode the payload according to that contract.
|
||||||
|
|
||||||
|
The [published JSON output contract](../integrations/json-output.md) defines
|
||||||
|
the index and bundle layout. Treat all paths obtained from a decoded external
|
||||||
|
document as untrusted until confined to their documented root.
|
||||||
|
|
||||||
|
## Complete Artifact Inventory
|
||||||
|
|
||||||
|
When every configured lane is accepted, the complete example publishes these
|
||||||
|
lane artifacts:
|
||||||
|
|
||||||
|
| Lane ID | Purpose | Canonical contract |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
|
||||||
|
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
|
||||||
|
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
|
||||||
|
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
|
||||||
|
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
|
||||||
|
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
|
||||||
|
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
|
||||||
|
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
|
||||||
|
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
|
||||||
|
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
|
||||||
|
|
||||||
|
The JSON encoder always publishes these bundle-management files:
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
|
||||||
|
| `manifest.json` | Run provenance and result summaries. |
|
||||||
|
| `rejected.json` | Rejected pipeline outputs. |
|
||||||
|
| `warnings.json` | Actionable process-degradation warnings. |
|
||||||
|
| `diagnostics.json` | Advisory and observation findings for accepted artifacts. |
|
||||||
|
|
||||||
|
The complete configuration also requests two pipeline-wide artifacts:
|
||||||
|
|
||||||
|
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
|
||||||
|
chunk metadata; and
|
||||||
|
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
|
||||||
|
excerpt containing the union of selected cited source units and the
|
||||||
|
configured surrounding window.
|
||||||
|
|
||||||
|
Discover both from their top-level `index.json` descriptors rather than
|
||||||
|
treating them as lanes. Evidence context is convenient reading material, not
|
||||||
|
authoritative provenance; citations in the normalized lane payloads remain the
|
||||||
|
evidence contract.
|
||||||
|
|
||||||
|
Every optional or lane file is published only when its corresponding artifact
|
||||||
|
is available. A successful process does not guarantee that all configured
|
||||||
|
lanes were accepted.
|
||||||
|
|
||||||
|
## Decide What Counts As Consumer Success
|
||||||
|
|
||||||
|
Exit status 0 means Notarius completed the pipeline and published its result
|
||||||
|
bundle. The receipt or bundle may still report warnings, rejected outputs, or
|
||||||
|
missing lane descriptors. A downstream consumer must define its own required
|
||||||
|
artifact set explicitly.
|
||||||
|
|
||||||
|
A caller that claims to consume the complete D&D workflow should normally
|
||||||
|
require all ten lane IDs in the table and verify each descriptor's expected
|
||||||
|
contract. If any required lane is missing, rejected, or incompatible, fail the
|
||||||
|
caller's extraction step while retaining the Notarius bundle for diagnosis. A
|
||||||
|
consumer that needs only a subset may define and document a narrower policy.
|
||||||
|
|
||||||
|
Keep the successful receipt with the complete published bundle. Retain
|
||||||
|
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
|
||||||
|
required by the caller's provenance, diagnosis, and retention policies. Avoid
|
||||||
|
selectively copying payload files without also preserving enough index and
|
||||||
|
manifest information to identify their originating run and contracts.
|
||||||
|
|
||||||
|
The transcript, lane artifacts, evidence context, manifest, debug data, and
|
||||||
|
logs can all contain private campaign information. Apply the same access,
|
||||||
|
publication, and retention controls used for the source transcript.
|
||||||
|
|
||||||
|
## Consumer Checklist
|
||||||
|
|
||||||
|
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
|
||||||
|
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
|
||||||
|
- Use absolute configuration, input, output-root, profile, and reference paths
|
||||||
|
in service deployments.
|
||||||
|
- Capture stdout and stderr separately and enforce cancellation and timeout.
|
||||||
|
- Parse stdout only after exit status 0.
|
||||||
|
- Accept only supported receipt, index, and artifact schema versions while
|
||||||
|
tolerating permitted unknown fields.
|
||||||
|
- Use the receipt's `output_directory`; never guess the run directory.
|
||||||
|
- Confine `index_file` and every descriptor path to the published bundle root.
|
||||||
|
- Discover lanes by `lane_id` and verify descriptor compatibility before
|
||||||
|
decoding payloads.
|
||||||
|
- Enforce an explicit required-lane policy and inspect rejections and warnings.
|
||||||
|
- Preserve the receipt and sufficient bundle provenance for every retained
|
||||||
|
artifact.
|
||||||
|
- Protect all transcript-derived files and diagnostic streams as sensitive
|
||||||
|
campaign data.
|
||||||
@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
|
|||||||
[Published JSON Output contract](../integrations/json-output.md) own the
|
[Published JSON Output contract](../integrations/json-output.md) own the
|
||||||
durable result formats.
|
durable result formats.
|
||||||
|
|
||||||
|
For the maintained complete D&D workflow, including its transcript input,
|
||||||
|
configured lane inventory, and downstream acceptance checklist, see
|
||||||
|
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
|
||||||
|
|
||||||
## Run And Check The Process
|
## Run And Check The Process
|
||||||
|
|
||||||
Optionally preflight a selected configuration and pipeline before work starts:
|
Optionally preflight a selected configuration and pipeline before work starts:
|
||||||
@@ -55,23 +59,30 @@ contract. The JSON bundle contract links to the available lane contracts.
|
|||||||
If `index.json` has an `evidence_context` descriptor, treat it as a
|
If `index.json` has an `evidence_context` descriptor, treat it as a
|
||||||
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
||||||
fields before decoding the linked file according to the [Published Evidence
|
fields before decoding the linked file according to the [Published Evidence
|
||||||
Context contract](../integrations/evidence-context.md). Use each
|
Context contract](../integrations/evidence-context.md). Decode its top-level
|
||||||
`evidence_refs` entry as the citation to source material. Its surrounding
|
source-unit array as a reading excerpt. Obtain authoritative citations and lane
|
||||||
context range and included units explain the citation, but do not widen or
|
provenance from the normalized lane artifacts; the excerpt has neither and its
|
||||||
replace the cited source reference.
|
nearby units do not widen a lane artifact's cited source reference.
|
||||||
|
|
||||||
A zero exit status may still report rejected outputs, warnings, or absent
|
A zero exit status may still report rejected outputs, warnings, or absent
|
||||||
lanes. The caller decides which lane IDs are required for its own work and
|
lanes. The caller decides which lane IDs are required for its own work and
|
||||||
which are optional; it should make that decision explicitly rather than infer
|
which are optional; it should make that decision explicitly rather than infer
|
||||||
failure from the receipt counts alone.
|
failure from the receipt counts alone.
|
||||||
|
|
||||||
|
When complete validation is required, also require receipt
|
||||||
|
`validation_status: approved` and inspect `validation_summaries`. A successful
|
||||||
|
run with `validation_status: incomplete` contains a structurally valid result
|
||||||
|
that advanced after validator execution could not complete under the configured
|
||||||
|
`warn_continue` policy. It is not reusable checkpoint state and should not be
|
||||||
|
silently treated as fully reviewed by the caller.
|
||||||
|
|
||||||
## Preserve Provenance And Handle Data Carefully
|
## Preserve Provenance And Handle Data Carefully
|
||||||
|
|
||||||
Keep the receipt with the published `manifest.json`, and retain
|
Keep the receipt with the published `manifest.json`, and retain
|
||||||
`rejected.json` and `warnings.json` when review or later provenance requires
|
`rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires
|
||||||
them. Treat the input, output bundle, cache, debug bundle, and captured process
|
them. Treat the input, output bundle, cache, debug bundle, and captured process
|
||||||
logs as potentially sensitive data. Apply the caller's access controls and
|
logs as potentially sensitive data. Apply the caller's access controls and
|
||||||
retention policy, and avoid copying secrets into arguments, logs, or
|
retention policy, and avoid copying secrets into arguments, logs, or
|
||||||
provenance records. An evidence-context artifact contains source-unit text and
|
provenance records. An evidence-context artifact contains source-unit text and
|
||||||
metadata, and selected lanes can cover most of an input; preserve and share it
|
metadata and can cover most of an input; preserve and share it only when that
|
||||||
only when that source content is authorized for the recipient.
|
source content is authorized for the recipient.
|
||||||
|
|||||||
@@ -18,13 +18,14 @@ implemented component map.
|
|||||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||||
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
||||||
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
||||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
|
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
|
||||||
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
||||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||||
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
||||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
||||||
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||||
|
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
|
||||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||||
|
|
||||||
For an existing subsystem, also inspect its focused tests and the package-local
|
For an existing subsystem, also inspect its focused tests and the package-local
|
||||||
|
|||||||
@@ -67,6 +67,12 @@ including a collision with the embedded catalog. Matching uses the catalog’s
|
|||||||
case, whitespace, and apostrophe normalization, so authors should avoid names
|
case, whitespace, and apostrophe normalization, so authors should avoid names
|
||||||
or aliases that normalize to another spell.
|
or aliases that normalize to another spell.
|
||||||
|
|
||||||
|
Spell extraction receives the effective catalog as deterministic canonical-name
|
||||||
|
and alias pairs. An alias in the transcript selects its associated canonical
|
||||||
|
name; the extractor is instructed to return that canonical spelling. The
|
||||||
|
projection contains no catalog source metadata or provenance, and aliases
|
||||||
|
remain recognition context rather than transcript evidence.
|
||||||
|
|
||||||
The overlay is a recognition aid only. The durable spell-artifact schema and
|
The overlay is a recognition aid only. The durable spell-artifact schema and
|
||||||
source-evidence rules are defined by the
|
source-evidence rules are defined by the
|
||||||
[D&D spell artifact contract](dnd-spell-artifacts.md).
|
[D&D spell artifact contract](dnd-spell-artifacts.md).
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# Published Evidence Context
|
# Published Evidence Context
|
||||||
|
|
||||||
This contract defines the optional `source/evidence-context` artifact emitted
|
This contract defines the optional `source/evidence-context` artifact emitted
|
||||||
by the production JSON output. Its configuration is owned by
|
by the production JSON output. It is a selected source-unit excerpt for
|
||||||
[Configuration](../config.md#module-bindings-and-validators); its logical-file
|
convenient reading alongside normalized lane artifacts; it is not a second
|
||||||
discovery is owned by [Published JSON Output](json-output.md).
|
citation or provenance model. Its configuration is owned by
|
||||||
|
[Configuration](../config.md#module-bindings-and-validators), and its
|
||||||
|
logical-file discovery is owned by [Published JSON Output](json-output.md).
|
||||||
|
|
||||||
## Identity And Discovery
|
## Identity And Discovery
|
||||||
|
|
||||||
@@ -26,35 +28,13 @@ its absence means evidence publication was not enabled for that bundle.
|
|||||||
|
|
||||||
## Payload
|
## Payload
|
||||||
|
|
||||||
The v1 payload is a JSON object with required `source_id`, `source_digest`,
|
The v1 payload is a top-level JSON array of generic source units. There is no
|
||||||
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
|
wrapper, source-level metadata, context grouping, lane identifier, or evidence
|
||||||
`contexts` are always arrays; an enabled configuration with no accepted direct
|
reference in the payload. An enabled configuration with no contributing
|
||||||
evidence publishes `contexts: []`.
|
accepted evidence publishes `[]`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
[
|
||||||
"source_id": "session-alpha",
|
|
||||||
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
|
||||||
"window_units": 1,
|
|
||||||
"selected_lanes": ["npc_registry", "spells"],
|
|
||||||
"contexts": [
|
|
||||||
{
|
|
||||||
"context_ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 20
|
|
||||||
},
|
|
||||||
"evidence_refs": [
|
|
||||||
{
|
|
||||||
"lane_id": "spells",
|
|
||||||
"source_ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"units": [
|
|
||||||
{
|
{
|
||||||
"id": 10,
|
"id": 10,
|
||||||
"kind": "transcript_segment",
|
"kind": "transcript_segment",
|
||||||
@@ -76,41 +56,52 @@ evidence publishes `contexts: []`.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Each context requires a `context_ref` object and `evidence_refs` and `units`
|
Each source unit has required `id`, `kind`, `text`, and self `ref` fields.
|
||||||
arrays. `context_ref` identifies the first and last included unit. Each
|
`ref` contains `source_id`, `start_unit_id`, and `end_unit_id`, and both unit
|
||||||
evidence entry contains a selected `lane_id` and an original `source_ref`. A
|
endpoints identify that unit's `id`. A unit may also contain source-owned
|
||||||
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
|
`metadata`, an open-ended JSON object. Fixed unit and reference fields are
|
||||||
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
|
strict: consumers must reject unknown fixed fields, malformed units, invalid
|
||||||
unknown fields; unit metadata may contain application-defined JSON values.
|
self-references, units whose `source_id` differs from other units in the same
|
||||||
|
excerpt, and a payload that is not the array described here.
|
||||||
|
|
||||||
## Citations And Context
|
The excerpt preserves each selected unit exactly as represented by the
|
||||||
|
validated generic source document. It does not add evidence-context-specific
|
||||||
|
annotations or reshape source-owned metadata.
|
||||||
|
|
||||||
`evidence_refs` are the authoritative citations. They identify the direct
|
## Selection And Citations
|
||||||
references emitted by accepted normalized artifacts. `context_ref` and the
|
|
||||||
units collection include those cited units plus nearby source units selected by
|
|
||||||
the configured window. They are explanatory context, not widened citations.
|
|
||||||
|
|
||||||
Only accepted outputs from the configured lane allowlist contribute. Rejected,
|
The framework obtains direct source references only through typed evidence
|
||||||
failed, absent, and lane-filtered outputs do not contribute. The artifact never
|
projections of accepted normalized artifacts in the configured lane allowlist.
|
||||||
contains raw input bytes, prompts, model responses, auxiliary reference
|
It validates each reference against the current source document, expands its
|
||||||
content, credentials, or filesystem paths.
|
range by `window_units` source-unit positions on each side, clamps at document
|
||||||
|
boundaries, and takes the union of all expanded ranges. The output contains
|
||||||
|
each selected source unit once in source-document position order, regardless
|
||||||
|
of numeric unit IDs. Repeated references, overlapping windows, and citations
|
||||||
|
from multiple lanes do not duplicate a unit. Rejected, failed, absent,
|
||||||
|
inactive, and unselected lanes contribute nothing.
|
||||||
|
|
||||||
## Ordering And Compatibility
|
Normalized lane artifacts remain authoritative for citations and for which lane
|
||||||
|
cited a range. The excerpt has no lane attribution and must not be used to
|
||||||
|
reconstruct it. Its included nearby units provide reading context only; they
|
||||||
|
do not widen any citation in a lane artifact.
|
||||||
|
|
||||||
The selected lane allowlist is lexical. Contexts and units are in source
|
The excerpt contains at most every generic source unit once. It can therefore
|
||||||
document position order, not numeric unit-ID order. Direct evidence entries
|
equal the complete generic source document when coverage is broad or the
|
||||||
are deterministically ordered by lane and source reference. Overlapping or
|
window is large. No byte-, token-, or compression-size guarantee is made, and
|
||||||
contiguous windows merge, and each source unit appears at most once in the
|
the framework does not truncate the excerpt to meet an arbitrary size limit.
|
||||||
resulting contexts.
|
|
||||||
|
## Consumer Responsibilities And Data Handling
|
||||||
|
|
||||||
The artifact is additive to the JSON bundle and is not a lane payload,
|
The artifact is additive to the JSON bundle and is not a lane payload,
|
||||||
normalized-output count, checkpoint, or generated reference. Consumers that
|
normalized-output count, checkpoint, or generated reference. Consumers that
|
||||||
do not need it must tolerate the absent optional descriptor. Consumers that do
|
do not need it must tolerate an absent descriptor. Consumers that do use it
|
||||||
use it should preserve the artifact and its schema identity with the run
|
should validate the descriptor and payload before use, retain the artifact with
|
||||||
provenance, and should treat its source text and metadata as sensitive durable
|
its schema identity when needed for a run record, and read citations from the
|
||||||
content.
|
corresponding normalized lane artifacts.
|
||||||
|
|
||||||
|
The excerpt contains source-unit text and source-owned metadata and is durable
|
||||||
|
output. Treat it as sensitive source content, apply appropriate access controls
|
||||||
|
and retention, and do not assume its selected form is materially smaller or
|
||||||
|
less sensitive than the original input.
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo
|
|||||||
## Bundle Layout
|
## Bundle Layout
|
||||||
|
|
||||||
All paths below are logical, relative, slash-separated bundle paths. The
|
All paths below are logical, relative, slash-separated bundle paths. The
|
||||||
encoder always emits the first four JSON files below and adds lane or
|
encoder always emits the first five JSON files below and adds lane or
|
||||||
pipeline-wide artifact files when their corresponding artifacts are available:
|
pipeline-wide artifact files when their corresponding artifacts are available:
|
||||||
|
|
||||||
A subprocess caller first obtains the physical bundle root from the
|
A subprocess caller first obtains the physical bundle root from the
|
||||||
@@ -21,10 +21,11 @@ root for the logical discovery described here.
|
|||||||
| `index.json` | Entry point that names the other published files and lane payloads. |
|
| `index.json` | Entry point that names the other published files and lane payloads. |
|
||||||
| `manifest.json` | Run provenance and result summaries. |
|
| `manifest.json` | Run provenance and result summaries. |
|
||||||
| `rejected.json` | Rejected pipeline outputs. |
|
| `rejected.json` | Rejected pipeline outputs. |
|
||||||
| `warnings.json` | Accepted-output and run warnings. |
|
| `warnings.json` | Actionable process-degradation warnings. |
|
||||||
|
| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. |
|
||||||
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
||||||
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
||||||
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
|
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
|
||||||
|
|
||||||
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
||||||
accepted only when their media type is `application/json`.
|
accepted only when their media type is `application/json`.
|
||||||
@@ -39,7 +40,8 @@ normalized lanes has this valid minimal index:
|
|||||||
"manifest_file": "manifest.json",
|
"manifest_file": "manifest.json",
|
||||||
"output_files": [],
|
"output_files": [],
|
||||||
"rejected_file": "rejected.json",
|
"rejected_file": "rejected.json",
|
||||||
"warnings_file": "warnings.json"
|
"warnings_file": "warnings.json",
|
||||||
|
"diagnostics_file": "diagnostics.json"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -49,6 +51,7 @@ normalized lanes has this valid minimal index:
|
|||||||
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
|
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
|
||||||
| `rejected_file` | Yes | Always `rejected.json`. |
|
| `rejected_file` | Yes | Always `rejected.json`. |
|
||||||
| `warnings_file` | Yes | Always `warnings.json`. |
|
| `warnings_file` | Yes | Always `warnings.json`. |
|
||||||
|
| `diagnostics_file` | Yes | Always `diagnostics.json`. |
|
||||||
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
|
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
|
||||||
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
|
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
|
||||||
|
|
||||||
@@ -92,7 +95,7 @@ group into the following externally observable summaries:
|
|||||||
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
||||||
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
||||||
| Source and references | `source_digests`, `references` |
|
| Source and references | `source_digests`, `references` |
|
||||||
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
|
| Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
|
||||||
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
||||||
|
|
||||||
`references` records provenance such as the target, slot, origin, digest,
|
`references` records provenance such as the target, slot, origin, digest,
|
||||||
@@ -102,6 +105,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
|
|||||||
provenance for the plan used by this run; cache records, debug artifacts, and
|
provenance for the plan used by this run; cache records, debug artifacts, and
|
||||||
other operational state are not published as bundle files.
|
other operational state are not published as bundle files.
|
||||||
|
|
||||||
|
Each `validation_summaries` entry is a bounded outcome for one producer result.
|
||||||
|
It has required `status`, `producer_attempt_count`, and `terminal_action`;
|
||||||
|
the stage and affected step, lane, module, or chunk identity are present when
|
||||||
|
applicable. `status` is `complete`, `rejected`, or `incomplete`.
|
||||||
|
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
|
||||||
|
configured validator order and omit later duplicates. Entries contain no raw
|
||||||
|
candidate response, correction guidance, validator diagnostic message, or
|
||||||
|
artifact payload. The same shape may appear as `validation` on an affected
|
||||||
|
rejection entry.
|
||||||
|
|
||||||
When present, `metadata.session_id` is the effective non-secret routing
|
When present, `metadata.session_id` is the effective non-secret routing
|
||||||
correlation identifier used for the run. It can be visible to providers and is
|
correlation identifier used for the run. It can be visible to providers and is
|
||||||
not a substitute for a cache or checkpoint identity. Its generation and
|
not a substitute for a cache or checkpoint identity. Its generation and
|
||||||
@@ -122,16 +135,57 @@ These values describe observed execution; they are not a backend-registration
|
|||||||
interface. Entries that differ by backend or effective reasoning remain
|
interface. Entries that differ by backend or effective reasoning remain
|
||||||
distinct even when their profile, provider, and model are otherwise equal.
|
distinct even when their profile, provider, and model are otherwise equal.
|
||||||
|
|
||||||
## Rejections And Warnings
|
## Rejections, Warnings, And Diagnostics
|
||||||
|
|
||||||
`rejected.json` is always an object with a `rejected` array. Each entry has
|
`rejected.json` is always an object with a `rejected` array. Each entry has
|
||||||
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
||||||
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
||||||
`diagnostic_artifact_path` are present only when applicable.
|
`diagnostic_artifact_path` are present only when applicable. An entry may also
|
||||||
|
contain the bounded `validation` summary described above; the existing singular
|
||||||
|
validator and reason fields remain the first configured rejection for
|
||||||
|
compatibility.
|
||||||
|
|
||||||
`warnings.json` is always an object with a `warnings` array. Each warning has
|
`warnings.json` is always the `notarius.warnings.v2` envelope:
|
||||||
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
|
|
||||||
there is nothing to report.
|
```json
|
||||||
|
{
|
||||||
|
"schema_version": "notarius.warnings.v2",
|
||||||
|
"group_count": 0,
|
||||||
|
"occurrence_count": 0,
|
||||||
|
"groups": []
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
It contains only process warnings. `group_count` is exact, and
|
||||||
|
`occurrence_count` is the exact sum of its group occurrence counts.
|
||||||
|
|
||||||
|
`diagnostics.json` is always the `notarius.diagnostics.v1` envelope:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema_version": "notarius.diagnostics.v1",
|
||||||
|
"group_count": 0,
|
||||||
|
"occurrence_count": 0,
|
||||||
|
"truncated": false,
|
||||||
|
"unrepresented_occurrence_count": 0,
|
||||||
|
"groups": []
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
It contains only advisory and observation groups. `group_count` counts groups
|
||||||
|
represented in `groups`; `occurrence_count` includes both represented and
|
||||||
|
unrepresented occurrences. When `truncated` is true,
|
||||||
|
`unrepresented_occurrence_count` is the exact number omitted from group
|
||||||
|
representation.
|
||||||
|
|
||||||
|
Each group has `disposition`, `category`, `reason_code`, framework-owned
|
||||||
|
`origin`, exact `occurrence_count`, bounded `samples`, and
|
||||||
|
`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk
|
||||||
|
ID and zero-based chunk index when applicable. A group retains at most three
|
||||||
|
distinct samples. The framework fails rather than truncating actionable
|
||||||
|
warnings beyond 128 groups; it represents at most 256 advisory/observation
|
||||||
|
groups and records further occurrences through the diagnostic truncation
|
||||||
|
fields above.
|
||||||
|
|
||||||
## Compatibility
|
## Compatibility
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# PromptKit Integration
|
# PromptKit Integration
|
||||||
|
|
||||||
Notarius pins
|
Notarius pins
|
||||||
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
|
[`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
|
||||||
as its in-process prompt engine. The upstream
|
as its in-process prompt engine. The upstream
|
||||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
|
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
owns the public engine API, and the upstream
|
owns the public engine API, and the upstream
|
||||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
owns prompt, profile, and schema file contracts.
|
owns prompt, profile, and schema file contracts.
|
||||||
|
|
||||||
## Supported Boundary
|
## Supported Boundary
|
||||||
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
||||||
operator and application-fallback profile sources;
|
operator and application-fallback profile sources;
|
||||||
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
||||||
variables, a direct session ID, prompt identity, and profile selection, then
|
variables, a direct session ID, prompt identity, profile selection, and
|
||||||
|
optional appended rendered messages, then
|
||||||
record credential-redacted details and run that exact execution;
|
record credential-redacted details and run that exact execution;
|
||||||
- return rendered debug material, validated structured output, selected
|
- return rendered debug material, validated structured output, selected
|
||||||
profile, backend, effective model metadata, and token usage;
|
profile, backend, effective model metadata, and token usage;
|
||||||
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
admission exhaustion through `ErrCapacityExceeded`.
|
admission exhaustion through `ErrCapacityExceeded`.
|
||||||
|
|
||||||
The pinned
|
The pinned
|
||||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
|
||||||
owns the registration and backend-capacity contract.
|
owns the registration and backend-capacity contract.
|
||||||
|
|
||||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||||
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
|
|||||||
field, which is authoritative for provider session behavior. It also retains
|
field, which is authoritative for provider session behavior. It also retains
|
||||||
the same value as the `session_id` prompt variable for maintained prompt
|
the same value as the `session_id` prompt variable for maintained prompt
|
||||||
compatibility. The generated identifier is 76 ASCII characters, within
|
compatibility. The generated identifier is 76 ASCII characters, within
|
||||||
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret
|
PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
|
||||||
correlation identifiers and may be exposed to providers and provider
|
correlation identifiers and may be exposed to providers and provider
|
||||||
observability. The CLI contract owns generation and override behavior.
|
observability. The CLI contract owns generation and override behavior.
|
||||||
|
|
||||||
@@ -84,7 +85,39 @@ configuration and deployment workflow are defined in
|
|||||||
[Configuration](../config.md#promptkit-profiles) and
|
[Configuration](../config.md#promptkit-profiles) and
|
||||||
[Operations](../operations.md#promptkit-profile-deployment).
|
[Operations](../operations.md#promptkit-profile-deployment).
|
||||||
|
|
||||||
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
|
PromptKit owns `base_profile` resolution under its
|
||||||
|
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
|
Notarius records the selected leaf identity and resolved target without parsing
|
||||||
|
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||||
|
reach the provider without authorization, which can result in a 401 or 403.
|
||||||
|
|
||||||
|
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
|
||||||
|
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
||||||
|
Maintained Notarius prompt definitions use only `system` and `user`.
|
||||||
|
|
||||||
|
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
|
||||||
|
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
|
||||||
|
exactly two messages in order: the latest validated producer response with
|
||||||
|
role `assistant`, then deterministic validation guidance with role `user`.
|
||||||
|
It never exposes a general caller-selected role API, accumulates earlier
|
||||||
|
correction turns, or changes the ordinary prompt prefix. Ordinary requests
|
||||||
|
leave appended messages unset.
|
||||||
|
|
||||||
|
PromptKit preserves supplied content but does not own Notarius's correction
|
||||||
|
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
|
||||||
|
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
|
||||||
|
1,114,112 bytes) before preparing the request. The transport-neutral
|
||||||
|
application contract owns defensive copying and these limits. Default request
|
||||||
|
and terminal summaries retain only safe counts, digests, identities, and usage;
|
||||||
|
complete appended messages remain limited to the explicitly requested detailed
|
||||||
|
debug trace.
|
||||||
|
|
||||||
|
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
|
||||||
|
profile catalogs from independently versioned transitive modules. Notarius
|
||||||
|
does not import or register either catalog; PromptKit retains catalog source,
|
||||||
|
identity, precedence, credential, and capacity ownership.
|
||||||
|
|
||||||
|
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
|
||||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||||
@@ -97,6 +130,10 @@ pinned upstream documentation.
|
|||||||
module assets, maps its transport-neutral completion contract, prepares and
|
module assets, maps its transport-neutral completion contract, prepares and
|
||||||
executes requests, validates output, records provenance, captures debug
|
executes requests, validates output, records provenance, captures debug
|
||||||
material, redacts errors, and preserves timeout ownership.
|
material, redacts errors, and preserves timeout ownership.
|
||||||
|
|
||||||
|
PromptKit provider error details do not cross the ordinary completion boundary.
|
||||||
|
Notarius exposes a provider-neutral generation category and optional status;
|
||||||
|
redacted provider details are retained only in requested debug material.
|
||||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||||
@@ -105,4 +142,7 @@ the conventional local backend.
|
|||||||
|
|
||||||
PromptKit API or format changes outside this boundary are not implicitly
|
PromptKit API or format changes outside this boundary are not implicitly
|
||||||
supported. Updating the pinned version requires reviewing the adapter and
|
supported. Updating the pinned version requires reviewing the adapter and
|
||||||
profile/configuration contracts against the upstream documentation.
|
profile/configuration contracts against the upstream documentation. Maintained
|
||||||
|
production prompts use PromptKit's bounded structural-repair contract; their
|
||||||
|
current declaration is one additional repair attempt. Notarius retains the
|
||||||
|
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||||
|
|||||||
@@ -9,36 +9,58 @@ Command syntax, streams, and exit statuses are defined in the
|
|||||||
|
|
||||||
## Schema
|
## Schema
|
||||||
|
|
||||||
The current schema version is `notarius.run-result.v1`.
|
The current schema version is `notarius.run-result.v2`.
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. |
|
| `schema_version` | Yes | Exactly `notarius.run-result.v2`. |
|
||||||
| `run_id` | Yes | The finalized Notarius run identifier. |
|
| `run_id` | Yes | The finalized Notarius run identifier. |
|
||||||
| `pipeline_id` | Yes | The effective pipeline identifier. |
|
| `pipeline_id` | Yes | The effective pipeline identifier. |
|
||||||
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
|
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
|
||||||
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
|
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
|
||||||
| `normalized_output_count` | Yes | Number of final normalized outputs. |
|
| `normalized_output_count` | Yes | Number of final normalized outputs. |
|
||||||
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
||||||
| `warning_count` | Yes | Number of final run warnings. |
|
| `warning_group_count` | Yes | Exact number of actionable warning groups. |
|
||||||
|
| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. |
|
||||||
|
| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. |
|
||||||
|
| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. |
|
||||||
|
| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. |
|
||||||
| `validation_status` | Yes | The final run manifest validation status. |
|
| `validation_status` | Yes | The final run manifest validation status. |
|
||||||
|
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
|
||||||
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
||||||
|
|
||||||
For the production `json` output module, `index_file` is present only when the
|
For the production `json` output module, `index_file` is present only when the
|
||||||
completed run returned exactly one logical output file named `index.json`.
|
completed run returned exactly one logical output file named `index.json`.
|
||||||
For another output module, its absence does not indicate a failed run.
|
For another output module, its absence does not indicate a failed run.
|
||||||
|
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
|
||||||
|
means one or more otherwise accepted results advanced under validator-failure
|
||||||
|
`warn_continue`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"schema_version": "notarius.run-result.v1",
|
"schema_version": "notarius.run-result.v2",
|
||||||
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||||
"pipeline_id": "dnd-session",
|
"pipeline_id": "dnd-session",
|
||||||
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||||
"index_file": "index.json",
|
"index_file": "index.json",
|
||||||
"normalized_output_count": 6,
|
"normalized_output_count": 6,
|
||||||
"rejected_output_count": 2,
|
"rejected_output_count": 2,
|
||||||
"warning_count": 1,
|
"warning_group_count": 1,
|
||||||
"validation_status": "rejected"
|
"warning_occurrence_count": 2,
|
||||||
|
"diagnostic_group_count": 3,
|
||||||
|
"diagnostic_occurrence_count": 5,
|
||||||
|
"diagnostics_truncated": false,
|
||||||
|
"validation_status": "incomplete",
|
||||||
|
"validation_summaries": [
|
||||||
|
{
|
||||||
|
"stage": "extract",
|
||||||
|
"lane_id": "spells",
|
||||||
|
"status": "incomplete",
|
||||||
|
"incomplete_validators": ["dnd/spells/source_refs"],
|
||||||
|
"producer_attempt_count": 1,
|
||||||
|
"terminal_action": "warn_continue"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -48,8 +70,11 @@ For another output module, its absence does not indicate a failed run.
|
|||||||
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
||||||
`output_directory` is the run-specific bundle, not the configured output root.
|
`output_directory` is the run-specific bundle, not the configured output root.
|
||||||
|
|
||||||
The receipt is a summary and discovery document. It does not contain lane
|
The receipt is a summary and discovery document. Its optional validation
|
||||||
descriptors, payloads, manifest data, rejections, warnings, or file contents.
|
summaries contain only stable status, identity, validator names, reason codes,
|
||||||
|
attempt counts, and terminal actions. It does not contain lane descriptors,
|
||||||
|
payloads, manifest payloads, rejection messages, warnings, raw model responses,
|
||||||
|
correction guidance, or file contents.
|
||||||
For the production JSON output, resolve `index_file` beneath
|
For the production JSON output, resolve `index_file` beneath
|
||||||
`output_directory`, reject path escapes, and use the
|
`output_directory`, reject path escapes, and use the
|
||||||
[Published JSON Output contract](json-output.md) to discover logical files and
|
[Published JSON Output contract](json-output.md) to discover logical files and
|
||||||
|
|||||||
@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
|
|||||||
|
|
||||||
## Dispatch And Configuration Handoff
|
## Dispatch And Configuration Handoff
|
||||||
|
|
||||||
The root dispatcher handles help, configuration validation, pipeline listing,
|
The root dispatcher handles help, version reporting, configuration validation,
|
||||||
and a pipeline run. It normalizes injectable options before dispatch so that a
|
pipeline listing, and a pipeline run. Version reporting resolves build
|
||||||
missing production dependency fails as a command error rather than reaching
|
information through `internal/buildinfo` before production composition, so the
|
||||||
execution.
|
diagnostic remains available without configuration or runtime collaborators.
|
||||||
|
The public syntax, streams, exit classes, and version semantics are defined by
|
||||||
|
the [CLI reference](../cli.md). Other root commands normalize injectable
|
||||||
|
options before dispatch so that a missing production dependency fails as a
|
||||||
|
command error rather than reaching execution.
|
||||||
|
|
||||||
Commands that need configuration use one shared loader. The CLI discovers the
|
Commands that need configuration use one shared loader. The CLI discovers the
|
||||||
file, parses it through **internal/core/config**, starts from defaults, applies
|
file, parses it through **internal/core/config**, starts from defaults, applies
|
||||||
|
|||||||
@@ -74,10 +74,19 @@ profile-free, and no second inheritance decision occurs during execution. The
|
|||||||
public field definitions and precedence are owned by
|
public field definitions and precedence are owned by
|
||||||
[Configuration](../config.md#pipelines).
|
[Configuration](../config.md#pipelines).
|
||||||
|
|
||||||
|
The resolver retains configured `validation_policy` overrides and derives one
|
||||||
|
detached concrete terminal policy for the chunk producer and every lane's
|
||||||
|
extract, merge, and normalize producers. That field-by-field inheritance is
|
||||||
|
complete before preparation, and the effective values contribute to pipeline
|
||||||
|
and checkpoint identity; execution does not interpret configuration defaults.
|
||||||
|
|
||||||
The framework resolver supplies defaults, selects lanes, resolves validator
|
The framework resolver supplies defaults, selects lanes, resolves validator
|
||||||
chains, checks registered module and artifact compatibility, validates module
|
chains, checks registered module and artifact compatibility, validates module
|
||||||
options, and returns the fixed ordered pipeline shape. The resulting
|
options, and returns the fixed ordered pipeline shape. Positive validator retry
|
||||||
**EffectiveConfig** retains the selected ID, requested selection and reference
|
budgets require an LLM-backed selected validator; deterministic validators are
|
||||||
|
rejected during resolution. Eligible LLM-backed producer specifications also
|
||||||
|
contribute their declared correction protocol to the resolved metadata. The
|
||||||
|
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
|
||||||
changes, a clone of the input configuration, and the resolved pipeline.
|
changes, a clone of the input configuration, and the resolved pipeline.
|
||||||
Callers may therefore retain or modify their input slices and maps without
|
Callers may therefore retain or modify their input slices and maps without
|
||||||
changing the resolved result, and later consumers cannot mutate the original
|
changing the resolved result, and later consumers cannot mutate the original
|
||||||
@@ -94,8 +103,8 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
|
|||||||
|
|
||||||
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
||||||
after defaults, lane selection, module bindings, reference bindings, validator
|
after defaults, lane selection, module bindings, reference bindings, validator
|
||||||
chains, effective LLM profiles, and artifact schema identity have been
|
chains, selected correction protocols, effective LLM profiles, and artifact
|
||||||
resolved. The digest excludes
|
schema identity have been resolved. The digest excludes
|
||||||
its own stored value. It identifies resolved composition rather than raw YAML
|
its own stored value. It identifies resolved composition rather than raw YAML
|
||||||
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
||||||
provenance before execution; cache and checkpoint identity have additional
|
provenance before execution; cache and checkpoint identity have additional
|
||||||
|
|||||||
@@ -33,7 +33,9 @@ typed builder. Scene chunking, every extractor, and NPC, location, and item-regi
|
|||||||
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
||||||
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
||||||
resolved-pipeline debug data and determines which selected bindings inherit the
|
resolved-pipeline debug data and determines which selected bindings inherit the
|
||||||
pipeline profile. Configuration remains the canonical owner of the exact keys,
|
pipeline profile. The registry normalizers use `single_response_v1`, forwarding
|
||||||
|
corrections to their reconciliation completion and retaining the accepted raw
|
||||||
|
proposal only as an owned model candidate. Configuration remains the canonical owner of the exact keys,
|
||||||
profile precedence, and validator order.
|
profile precedence, and validator order.
|
||||||
|
|
||||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||||
@@ -42,6 +44,13 @@ unknown fields, while preserving semantic candidates for deterministic
|
|||||||
validation. Do not promote a private response envelope into a durable schema;
|
validation. Do not promote a private response envelope into a durable schema;
|
||||||
the contracts above define durable data.
|
the contracts above define durable data.
|
||||||
|
|
||||||
|
A D&D producer that declares `single_response_v1` forwards any supplied
|
||||||
|
semantic correction to its structured completion and returns an owned copy of
|
||||||
|
that completion's exact validated raw response as its model candidate. It does
|
||||||
|
not serialize normalized artifacts to create that candidate, so deterministic
|
||||||
|
identity, evidence, warning, and durable-schema behavior remains separate from
|
||||||
|
the model transport material.
|
||||||
|
|
||||||
## Prompt Construction
|
## Prompt Construction
|
||||||
|
|
||||||
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
|
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
|
||||||
@@ -84,13 +93,15 @@ replace it with a complete profile of the same ID from the configured PromptKit
|
|||||||
source. Deployment profile selection is documented in
|
source. Deployment profile selection is documented in
|
||||||
[Configuration](../config.md#promptkit-profiles).
|
[Configuration](../config.md#promptkit-profiles).
|
||||||
|
|
||||||
The transcript assets have distinct consumers. Scene chunking consumes the
|
The D&D transcript assets have distinct consumers. Scene chunking consumes the
|
||||||
complete-session `common-dnd-transcript-full.md`; extraction prompts consume
|
complete-session `common-dnd-transcript-full.md`, while extraction prompts
|
||||||
the current-chunk `common-dnd-transcript-chunk.md`; and NPC, location, and item
|
consume the current-chunk `common-dnd-transcript-chunk.md`. NPC, location, and
|
||||||
normalization consume `common-dnd-transcript-windows.md` alongside their
|
item normalization instead mount the generic semantic-reconciliation
|
||||||
candidate collections. Player, party, glossary, and compatible campaign
|
candidate and transcript-window presentation assets. Player, party, glossary,
|
||||||
references provide disambiguating context, not evidence. Reference material is
|
and compatible campaign references provide disambiguating context only when
|
||||||
canonically ordered before rendering so equivalent inputs remain stable.
|
declared by the active prompt; they never establish evidence. Reference
|
||||||
|
material is canonically ordered before rendering so equivalent inputs remain
|
||||||
|
stable.
|
||||||
|
|
||||||
Extraction prompts render the common system and identity messages first, then
|
Extraction prompts render the common system and identity messages first, then
|
||||||
cached campaign references and the cached chunk transcript. Evidence policy and
|
cached campaign references and the cached chunk transcript. Evidence policy and
|
||||||
@@ -100,10 +111,11 @@ reusable extraction prefix identical while preserving the lane-specific suffix.
|
|||||||
|
|
||||||
Scene chunking intentionally uses a different order: system, cached campaign
|
Scene chunking intentionally uses a different order: system, cached campaign
|
||||||
references, uncached module instructions, then the final ephemeral full
|
references, uncached module instructions, then the final ephemeral full
|
||||||
transcript. Entity normalization also has its own order: system, uncached
|
transcript. Entity normalization also has its own order: D&D system, mandatory
|
||||||
module instructions, ephemeral reconciliation policy, uncached candidates, and
|
generic protocol, ephemeral domain semantic instructions, generic candidate
|
||||||
final ephemeral transcript windows. These orders and cache controls are prompt
|
presentation, and final ephemeral generic transcript windows. These orders and
|
||||||
behavior; change them only through the owning manifest and prompt declaration.
|
cache controls are prompt behavior; change them only through the owning
|
||||||
|
manifest and prompt declaration.
|
||||||
|
|
||||||
## Evidence, Candidates, And Normalization
|
## Evidence, Candidates, And Normalization
|
||||||
|
|
||||||
@@ -116,11 +128,19 @@ result.
|
|||||||
|
|
||||||
Default chains keep responsibilities separate: structural validators assess the
|
Default chains keep responsibilities separate: structural validators assess the
|
||||||
candidate, source-reference validators resolve cited ranges against the current
|
candidate, source-reference validators resolve cited ranges against the current
|
||||||
source, durable-schema validation checks an approved representation, and
|
source and require extraction evidence to stay within the current chunk,
|
||||||
|
durable-schema validation checks an approved representation, and
|
||||||
relatedness validators report advisory evidence concerns. The configured order
|
relatedness validators report advisory evidence concerns. The configured order
|
||||||
is documented in
|
is documented in
|
||||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||||
|
|
||||||
|
Every D&D rejection describes the correction in transcript-grounded domain
|
||||||
|
terms, using contextual names, artifact fields, and source segment ranges when
|
||||||
|
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
||||||
|
hashes, validator module keys, or reason codes. Those identifiers remain in
|
||||||
|
ordinary validation provenance; only the actionable semantic guidance is
|
||||||
|
eligible for the correction prompt.
|
||||||
|
|
||||||
Enemy-event extraction additionally rejects a second `engaged` observation for
|
Enemy-event extraction additionally rejects a second `engaged` observation for
|
||||||
the same comparison identity within one scene-scoped result. Normalization may
|
the same comparison identity within one scene-scoped result. Normalization may
|
||||||
combine results from distinct scenes, so it intentionally does not apply that
|
combine results from distinct scenes, so it intentionally does not apply that
|
||||||
@@ -129,15 +149,49 @@ rule. Configuration owns the exact validator key and chain position.
|
|||||||
Normalizers are deterministic for spells, combat turns, item occurrences, NPC
|
Normalizers are deterministic for spells, combat turns, item occurrences, NPC
|
||||||
occurrences, scene descriptions, enemy events, and location occurrences. They
|
occurrences, scene descriptions, enemy events, and location occurrences. They
|
||||||
canonicalize display values and evidence, use source-document order for stable
|
canonicalize display values and evidence, use source-document order for stable
|
||||||
output, and issue bounded warnings for changes or collapsed duplicates. NPC,
|
output, and emit bounded normalization observations for changes or collapsed duplicates. NPC,
|
||||||
item, and location registry normalizers are intentional exceptions: each first
|
item, and location registry normalizers are intentional exceptions: each first
|
||||||
produces a deterministic candidate set, then may use a bounded structured-LLM
|
produces a deterministic candidate set, then may use a bounded structured-LLM
|
||||||
proposal to reconcile identity groups. The proposal selects supplied
|
proposal to reconcile identity groups.
|
||||||
descriptors—names with their candidate source references—not durable IDs.
|
|
||||||
Request-local candidate keys may support resolution internally, but are never
|
## Semantic Registry Reconciliation
|
||||||
included in model input or output. Colliding descriptors are ineligible, and
|
|
||||||
invalid or unusable proposals retain the deterministic result with retry or
|
The three registry normalizers instantiate the domain-neutral
|
||||||
fallback diagnostics; the model does not directly replace durable records.
|
`internal/framework/semanticreconcile` engine with default bounds. Each
|
||||||
|
eligible candidate receives a contiguous, one-based `candidate_id` for that
|
||||||
|
request. The model sees that handle, the candidate label and source-free
|
||||||
|
evidence ranges, plus bounded transcript windows; it returns only duplicate
|
||||||
|
groups of supplied handles and one supplied canonical handle per group. It
|
||||||
|
never returns names, evidence, durable IDs, or replacement records. Identical
|
||||||
|
labels and evidence remain independently selectable because their handles are
|
||||||
|
distinct.
|
||||||
|
|
||||||
|
The generic core owns the mandatory handle protocol, candidate and transcript
|
||||||
|
presentation, the private response schema, source-reference validation,
|
||||||
|
candidate and combined-material limits, structured completion, proposal
|
||||||
|
assessment, stable group ordering, and typed plan-application mechanics. The
|
||||||
|
D&D prompt contributes its system message and registry-specific semantic
|
||||||
|
instructions. The generic registrar registers the shared prompt and schema;
|
||||||
|
the D&D registrar registers each consuming prompt and the fallback profile.
|
||||||
|
|
||||||
|
Fewer than two eligible candidates skips the LLM without a semantic warning.
|
||||||
|
An exceeded bound also skips the call and preserves the deterministic
|
||||||
|
preprocessed registry, adding the registry's bounded fallback warning. Invalid
|
||||||
|
structured output or discarded proposal groups use the normalizer's existing
|
||||||
|
retry contract; retry exhaustion preserves the safe deterministic or
|
||||||
|
partially applied result and emits its bounded fallback warning. Provider,
|
||||||
|
transport, cancellation, and context-material failures remain execution
|
||||||
|
errors.
|
||||||
|
|
||||||
|
Application remains typed and registry-owned. All three policies select the
|
||||||
|
canonical member's normalized display name, union member evidence in source
|
||||||
|
order, preserve ungrouped records, and derive durable identity only after
|
||||||
|
consolidation. NPC IDs derive from the final name. Item IDs also derive from
|
||||||
|
the final name, and a typed guard prevents currency aliases from crossing
|
||||||
|
denominations or mixing currency with non-currency records. Location IDs
|
||||||
|
derive from the final name and final evidence, preserving same-name,
|
||||||
|
parent/child, and distinct physical-place identities. Registry warning scopes,
|
||||||
|
reason codes, and postconditions remain outside the generic core.
|
||||||
|
|
||||||
## Generated References And Grounding
|
## Generated References And Grounding
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,22 @@ observability. The adapter returns PromptKit’s validated raw bytes rather than
|
|||||||
re-encoding the decoded target. An empty optional material is represented as
|
re-encoding the decoded target. An empty optional material is represented as
|
||||||
one space so its named input is retained by PromptKit.
|
one space so its named input is retained by PromptKit.
|
||||||
|
|
||||||
|
When a request includes semantic correction material, the adapter validates and
|
||||||
|
defensively copies it before preparation, then appends exactly two messages
|
||||||
|
after the ordinarily rendered prompt: the prior response as an assistant
|
||||||
|
message and the correction guidance as a user message. Requests without a
|
||||||
|
correction do not add messages or introduce caller roles. Ordinary request
|
||||||
|
summaries record correction byte counts and digests only; complete messages are
|
||||||
|
available solely in an explicitly requested debug trace.
|
||||||
|
|
||||||
|
The adapter leaves the ordinary rendered message prefix, named inputs,
|
||||||
|
variables, session, profile, execution overrides, prepared-execution path, and
|
||||||
|
PromptKit repair policy unchanged for a corrected request. It never imports a
|
||||||
|
PromptKit message type into a module or pipeline contract. PromptKit reports
|
||||||
|
actual structural repair count and cumulative token usage per completion; the
|
||||||
|
pipeline's safe terminal debug record projects those values without copying
|
||||||
|
message content.
|
||||||
|
|
||||||
Client construction may also receive a run-wide reasoning-effort override from
|
Client construction may also receive a run-wide reasoning-effort override from
|
||||||
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
||||||
creates a fresh PromptKit execution override for each request: a nil pointer
|
creates a fresh PromptKit execution override for each request: a nil pointer
|
||||||
@@ -75,7 +91,7 @@ backend membership as runtime without performing generation. Fallback assets
|
|||||||
are mounted only when at least one source is registered. The production D&D
|
are mounted only when at least one source is registered. The production D&D
|
||||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||||
profile parsing: an operator-provided matching profile takes precedence over a
|
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||||
fallback profile without Notarius merging either document.
|
fallback profile without Notarius merging either document.
|
||||||
When the registration is absent, a profile selecting `backend: local` fails
|
When the registration is absent, a profile selecting `backend: local` fails
|
||||||
inspection instead of falling back to a built-in or endpoint-only target.
|
inspection instead of falling back to a built-in or endpoint-only target.
|
||||||
@@ -102,7 +118,8 @@ because it changes scheduling rather than execution semantics.
|
|||||||
Production construction creates one PromptKit client and wraps it in one
|
Production construction creates one PromptKit client and wraps it in one
|
||||||
scheduled client. The scheduler has a fixed, positive permit limit, serves
|
scheduled client. The scheduler has a fixed, positive permit limit, serves
|
||||||
queued calls in FIFO order, and removes a queued call when its context is
|
queued calls in FIFO order, and removes a queued call when its context is
|
||||||
cancelled. A granted permit is released exactly once on every completion path.
|
cancelled. It rechecks the caller context after admission and before dispatch.
|
||||||
|
A granted permit is released exactly once on every completion path.
|
||||||
|
|
||||||
The scheduled wrapper surrounds every `CompleteStructured` call, so concurrent
|
The scheduled wrapper surrounds every `CompleteStructured` call, so concurrent
|
||||||
lanes, pipeline retries, and LLM-backed validators share the same provider-call
|
lanes, pipeline retries, and LLM-backed validators share the same provider-call
|
||||||
@@ -141,12 +158,28 @@ arrangement and its data-only boundary are defined by
|
|||||||
[ADR-0011](../adr/0011-centralize-llm-assets.md), rather than by this runtime
|
[ADR-0011](../adr/0011-centralize-llm-assets.md), rather than by this runtime
|
||||||
guide.
|
guide.
|
||||||
|
|
||||||
|
The generic registrar is the sole production registration owner for the
|
||||||
|
semantic-reconciliation default prompt and private response schema. The
|
||||||
|
domain-neutral reconciliation package also exposes only its mandatory protocol
|
||||||
|
and candidate/transcript presentation files for domain prompt manifests. D&D
|
||||||
|
registry normalizers mount those files while retaining ownership and hashing
|
||||||
|
of their D&D system message, semantic instructions, and complete prompt
|
||||||
|
declaration. The response schema is therefore registered once even though
|
||||||
|
several typed normalizers select it.
|
||||||
|
|
||||||
Mounted prompt assets determine a module's fingerprint. The fingerprint hashes
|
Mounted prompt assets determine a module's fingerprint. The fingerprint hashes
|
||||||
only the module and shared files explicitly selected by its manifest, so an
|
only the module and shared files explicitly selected by its manifest, so an
|
||||||
unrelated asset does not invalidate a checkpoint. Schema loaders validate JSON,
|
unrelated asset does not invalidate a checkpoint. Schema loaders validate JSON,
|
||||||
attach identity and digest metadata, make defensive copies, and expose
|
attach identity and digest metadata, make defensive copies, and expose
|
||||||
diagnostics without raw schema bytes.
|
diagnostics without raw schema bytes.
|
||||||
|
|
||||||
|
Semantic-reconciliation normalizers extend this identity with the shared
|
||||||
|
response-schema digest, framework policy version, and complete limit-policy
|
||||||
|
digest. Their manifest metadata records the same content-free prompt, schema,
|
||||||
|
policy, and limit identities together with domain identity and normalization
|
||||||
|
policies. Request-local handles, source material, proposal content, and raw
|
||||||
|
asset bytes are not checkpoint metadata.
|
||||||
|
|
||||||
Private response schemas validate a model transport envelope. They are not the
|
Private response schemas validate a model transport envelope. They are not the
|
||||||
durable artifact schema and should not be documented as an external wire
|
durable artifact schema and should not be documented as an external wire
|
||||||
contract. Durable formats and compatibility rules remain in the
|
contract. Durable formats and compatibility rules remain in the
|
||||||
@@ -179,7 +212,9 @@ structured-output validation. The adapter reports an empty result, validation
|
|||||||
failure, empty structured body, or decode failure as
|
failure, empty structured body, or decode failure as
|
||||||
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
|
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
|
||||||
material when they exist. Provider failures remain operational errors rather
|
material when they exist. Provider failures remain operational errors rather
|
||||||
than output-validation failures.
|
than output-validation failures. Apart from documented context, capacity, and
|
||||||
|
invalid-output categories, provider error values and types do not cross the
|
||||||
|
adapter error chain; callers receive only a credential-redacted diagnostic.
|
||||||
|
|
||||||
When PromptKit rejects backend admission before generation, the adapter maps
|
When PromptKit rejects backend admission before generation, the adapter maps
|
||||||
`promptkit.ErrCapacityExceeded` to
|
`promptkit.ErrCapacityExceeded` to
|
||||||
@@ -191,13 +226,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
|
|||||||
the pipeline's existing binding attempt policy sees the operational error and
|
the pipeline's existing binding attempt policy sees the operational error and
|
||||||
decides whether to rerun the complete operation.
|
decides whether to rerun the complete operation.
|
||||||
|
|
||||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
PromptKit executes structural repair within its structured-output flow. The
|
||||||
The current production D&D prompt manifests set repair attempts to zero. That
|
maintained production prompt manifests declare one additional repair attempt.
|
||||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
When a resolved binding supplies a repair value, the adapter inspects the
|
||||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
prompt, copies its complete output contract, changes only the repair limit, and
|
||||||
rejection is a recorded output rather than a provider error. The pipeline owns
|
passes that complete replacement contract to PromptKit. This preserves the
|
||||||
attempt lifecycle, validation chains, and retry diagnostics; see
|
prompt's output format, validation mode, schema, and provider structured-output
|
||||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
settings.
|
||||||
|
|
||||||
|
A successful repair is an ordinary successful completion, not a warning. The
|
||||||
|
adapter reports PromptKit's actual repair count and its cumulative usage
|
||||||
|
directly, without adding the initial and corrective counts again. Debug prompt
|
||||||
|
material records the configured complete contract; debug response material
|
||||||
|
records the repaired response and actual validation result. If the repair
|
||||||
|
budget is exhausted, the adapter retains the final raw bytes and debug material
|
||||||
|
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
|
||||||
|
or corrective call remain provider-neutral operational errors with the same
|
||||||
|
redaction boundary.
|
||||||
|
|
||||||
|
Structural repair does not replace pipeline retry behavior: a binding's
|
||||||
|
configured retry count reruns its complete stage attempt after an operational
|
||||||
|
or structural error, module-requested retry, or actionable semantic rejection.
|
||||||
|
The pipeline owns attempt lifecycle, validation chains, and retry diagnostics;
|
||||||
|
see [Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||||
[binding reference](../config.md#module-bindings-and-validators).
|
[binding reference](../config.md#module-bindings-and-validators).
|
||||||
|
|
||||||
## Timeout Ownership
|
## Timeout Ownership
|
||||||
@@ -228,6 +279,13 @@ surfaced; when the completion already failed, its call error remains the
|
|||||||
result. Debug-bundle location, retention, and handling are operational concerns
|
result. Debug-bundle location, retention, and handling are operational concerns
|
||||||
documented in [Operations](../operations.md#debug-bundles).
|
documented in [Operations](../operations.md#debug-bundles).
|
||||||
|
|
||||||
|
The attempt-terminal summary is a separate safe trace record: it contains
|
||||||
|
attempt kinds, validator outcome counts and reason codes, effective policy,
|
||||||
|
terminal action, and repair/usage references. It excludes raw assistant
|
||||||
|
responses and correction text. Those values can appear only in the explicitly
|
||||||
|
requested detailed prompt and response artifacts, which require sensitive-data
|
||||||
|
handling.
|
||||||
|
|
||||||
Run manifests receive selected profile summaries, including optional effective
|
Run manifests receive selected profile summaries, including optional effective
|
||||||
backend and reasoning provenance, and component identities—not prompt, schema,
|
backend and reasoning provenance, and component identities—not prompt, schema,
|
||||||
source, reference, or response content. The published field semantics belong
|
source, reference, or response content. The published field semantics belong
|
||||||
@@ -237,6 +295,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
|
|||||||
available to other runtime collaborators; it does not make prompt or response
|
available to other runtime collaborators; it does not make prompt or response
|
||||||
contents safe for general logging.
|
contents safe for general logging.
|
||||||
|
|
||||||
|
Generation failures expose an application-owned category and optional HTTP
|
||||||
|
status. Provider code, type, and message remain debug-only, after redaction.
|
||||||
|
|
||||||
## Failure Boundaries
|
## Failure Boundaries
|
||||||
|
|
||||||
- Construction fails for missing asset registries, mutually exclusive profile
|
- Construction fails for missing asset registries, mutually exclusive profile
|
||||||
|
|||||||
@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
|
|||||||
binding-specific profile on a deterministic module. The user-facing precedence
|
binding-specific profile on a deterministic module. The user-facing precedence
|
||||||
contract belongs in [Configuration](../config.md#pipelines).
|
contract belongs in [Configuration](../config.md#pipelines).
|
||||||
|
|
||||||
|
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
|
||||||
|
declare correction protocol `single_response_v1`. That declaration is a
|
||||||
|
promise that the implementation accepts one attempt-local semantic correction
|
||||||
|
and returns an owned copy of the exact one model response that directly
|
||||||
|
controlled the candidate. It must forward correction only to its structured
|
||||||
|
completion request; it must not manufacture prior-response material by
|
||||||
|
serializing a normalized artifact or expose opaque application IDs. Input,
|
||||||
|
output, validator, and deterministic specs cannot declare the protocol.
|
||||||
|
|
||||||
Implementations that accept options must provide both an option validator and
|
Implementations that accept options must provide both an option validator and
|
||||||
a builder. The validator is used while resolving configuration; the builder
|
a builder. The validator is used while resolving configuration; the builder
|
||||||
decodes the same options and constructs the implementation from the prepared
|
decodes the same options and constructs the implementation from the prepared
|
||||||
@@ -36,20 +45,38 @@ they need, register each leaf implementation, and add any family-owned assets
|
|||||||
or default validator chains. They return contextual errors so production
|
or default validator chains. They return contextual errors so production
|
||||||
composition fails at startup rather than at the first run.
|
composition fails at startup rather than at the first run.
|
||||||
|
|
||||||
|
A validator that returns a completed rejection must supply two separate
|
||||||
|
bounded values: a stable `ReasonCode` for provenance and actionable
|
||||||
|
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
|
||||||
|
and the constraints on one complete corrected replacement. It must not contain
|
||||||
|
validator keys, diagnostic paths, opaque application IDs, or other internal
|
||||||
|
identifiers. An operator-facing `Message` may explain the same event, but the
|
||||||
|
framework never copies it into a model request. Missing or invalid guidance is
|
||||||
|
a validator contract failure.
|
||||||
|
|
||||||
An artifact family can register an optional typed evidence projector alongside
|
An artifact family can register an optional typed evidence projector alongside
|
||||||
its codec. The projector returns defensive copies of the artifact's direct
|
its codec. The projector returns defensive copies of the artifact's direct
|
||||||
generic source references and must use the codec's exact Go type. It does not
|
generic source references and must use the codec's exact Go type. It does not
|
||||||
interpret surrounding context or publish files; the pipeline validates the
|
interpret surrounding context or publish files; the pipeline validates the
|
||||||
capability during preparation and the output boundary owns publication. See
|
capability during preparation and the output boundary owns publication. See
|
||||||
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
||||||
for the durable result.
|
for the durable source-unit excerpt. Lane artifacts retain citation and lane
|
||||||
|
provenance; the framework does not add either to that published excerpt.
|
||||||
|
|
||||||
|
An artifact family is broader than a module: it owns the cohesive domain
|
||||||
|
feature across its artifact type, codec, stage modules, validators, prompt
|
||||||
|
policy, schemas, identity helpers, and reference projections. An extractor and
|
||||||
|
normalizer in one artifact family remain independently registered modules in
|
||||||
|
their respective pipeline stages. This ownership vocabulary does not create a
|
||||||
|
new registry or change the fixed pipeline.
|
||||||
|
|
||||||
## Production Composition
|
## Production Composition
|
||||||
|
|
||||||
Production composition is intentionally split by family:
|
Production composition is intentionally split by family:
|
||||||
|
|
||||||
- The generic registrar provides the unit chunker, generic JSON validators,
|
- The generic registrar provides the unit chunker, generic JSON validators,
|
||||||
and JSON output encoder.
|
JSON output encoder, and shared semantic-reconciliation prompt and response
|
||||||
|
schema assets.
|
||||||
- The Seriatim registrar provides the transcript input adapter. Its external
|
- The Seriatim registrar provides the transcript input adapter. Its external
|
||||||
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
|
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
|
||||||
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
|
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
|
||||||
@@ -60,6 +87,42 @@ The CLI owns the composition that invokes these registrars. A module package
|
|||||||
may register its own family but must not assemble the CLI or make framework
|
may register its own family but must not assemble the CLI or make framework
|
||||||
packages depend on production extensions.
|
packages depend on production extensions.
|
||||||
|
|
||||||
|
## Semantic Reconciliation
|
||||||
|
|
||||||
|
`internal/framework/semanticreconcile` is a domain-neutral strategy used by a
|
||||||
|
typed normalize module; it is not itself a selectable stage module. A
|
||||||
|
source-backed artifact-family normalizer projects its deterministic records
|
||||||
|
into contextual candidates and owned typed record envelopes, supplies its
|
||||||
|
chosen prompt identity and resolved LLM profile, and constructs an engine with
|
||||||
|
explicit limits. The core filters invalid evidence, assigns contiguous
|
||||||
|
request-local integer handles, renders bounded candidate and transcript
|
||||||
|
materials, invokes the structured-completion boundary, and assesses the
|
||||||
|
returned duplicate groups into a stable non-overlapping plan.
|
||||||
|
|
||||||
|
The normalizer then applies that plan through a typed `ApplicationPolicy`. The
|
||||||
|
core preserves ungrouped records, contribution order, and provenance while the
|
||||||
|
artifact family owns group guards, field and evidence consolidation, durable
|
||||||
|
ID derivation, retry and fallback presentation, classified diagnostics, and postconditions.
|
||||||
|
Request-local handles do not enter the typed value or durable artifact. Fewer
|
||||||
|
than two eligible candidates skips model invocation; exceeding a candidate or
|
||||||
|
combined-material bound preserves the deterministic result under the family's
|
||||||
|
fallback policy. Provider, transport, cancellation, and context-construction
|
||||||
|
failures remain execution errors.
|
||||||
|
|
||||||
|
When the engine actually makes a proposal call, its typed result carries the
|
||||||
|
owned exact proposal response under the same correction contract as other
|
||||||
|
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
|
||||||
|
model candidate, so a later rejection applies terminal policy without spending
|
||||||
|
an ineffective semantic retry.
|
||||||
|
|
||||||
|
The core supplies a conservative generic prompt and the single private
|
||||||
|
response schema. A domain prompt may substitute its semantic instructions but
|
||||||
|
mounts the core-owned protocol and candidate/transcript presentation assets.
|
||||||
|
Prompt, schema, policy, and limit identities participate in manifest metadata
|
||||||
|
and checkpoint fingerprints. The generic registrar owns production
|
||||||
|
registration of those shared assets; a consuming domain registrar owns only
|
||||||
|
its domain prompt.
|
||||||
|
|
||||||
## Adding Or Changing A Module
|
## Adding Or Changing A Module
|
||||||
|
|
||||||
1. Choose the pipeline stage and the typed artifact boundary. Put external
|
1. Choose the pipeline stage and the typed artifact boundary. Put external
|
||||||
@@ -72,9 +135,13 @@ packages depend on production extensions.
|
|||||||
3. Implement strict option decoding, construction, and the typed stage
|
3. Implement strict option decoding, construction, and the typed stage
|
||||||
interface. Preserve caller ownership: do not retain mutable request data
|
interface. Preserve caller ownership: do not retain mutable request data
|
||||||
and return defensive copies where an implementation exposes stored data.
|
and return defensive copies where an implementation exposes stored data.
|
||||||
|
If declaring correction capability, forward the request correction and
|
||||||
|
retain only the exact validated response that controlled the result.
|
||||||
4. Register the module through its typed registry helper and add it to the
|
4. Register the module through its typed registry helper and add it to the
|
||||||
owning family registrar. Add a default validator chain only when that
|
owning family registrar. Add a default validator chain only when that
|
||||||
family owns the behavior; otherwise require an explicit compatible chain.
|
family owns the behavior; otherwise require an explicit compatible chain.
|
||||||
|
Every rejection path in a validator must provide actionable correction
|
||||||
|
guidance while retaining its stable internal reason code.
|
||||||
5. Update the selectable-key and chain reference in
|
5. Update the selectable-key and chain reference in
|
||||||
[Configuration](../config.md#production-module-keys), the applicable
|
[Configuration](../config.md#production-module-keys), the applicable
|
||||||
integration contract, and focused tests. Keep the configuration document
|
integration contract, and focused tests. Keep the configuration document
|
||||||
|
|||||||
@@ -25,10 +25,12 @@ physical state roots.
|
|||||||
| Area | Implemented owners | Responsibility |
|
| Area | Implemented owners | Responsibility |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
||||||
|
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
|
||||||
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
||||||
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
||||||
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
||||||
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
|
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
|
||||||
|
| Semantic reconciliation | **internal/framework/semanticreconcile** | Bounded source-backed candidate preparation, request-local handle proposals, deterministic assessment, typed plan application, and reconciliation identity metadata; see [Module Internals](modules.md#semantic-reconciliation) and [D&D Module Internals](dnd.md#semantic-registry-reconciliation). |
|
||||||
| Embedded LLM content | **assets** | Read-only centralized LLM-facing content, scoped by its consuming package; see [LLM Runtime](llm.md#prompt-and-schema-assets) and [D&D Module Internals](dnd.md#prompt-construction). |
|
| Embedded LLM content | **assets** | Read-only centralized LLM-facing content, scoped by its consuming package; see [LLM Runtime](llm.md#prompt-and-schema-assets) and [D&D Module Internals](dnd.md#prompt-construction). |
|
||||||
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
|
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
|
||||||
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
|
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
|
||||||
@@ -49,8 +51,9 @@ the CLI composition boundary.
|
|||||||
composition, and path safety.
|
composition, and path safety.
|
||||||
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
|
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
|
||||||
profiles, and secret handling.
|
profiles, and secret handling.
|
||||||
- [Module Internals](modules.md): generic extension registration, module
|
- [Module Internals](modules.md): generic extension registration, artifact
|
||||||
construction, validation, and reference mechanics.
|
families, module construction, semantic reconciliation, validation, and
|
||||||
|
reference mechanics.
|
||||||
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
|
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
|
||||||
reference projections, and lane-specific exceptions. Durable D&D and
|
reference projections, and lane-specific exceptions. Durable D&D and
|
||||||
Seriatim data shapes remain in the [integration contracts](../integrations/).
|
Seriatim data shapes remain in the [integration contracts](../integrations/).
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ own durable output shapes. Concrete production extensions are covered by
|
|||||||
The pipeline framework accepts a resolved composition, registries, shared
|
The pipeline framework accepts a resolved composition, registries, shared
|
||||||
dependencies, input bytes, a supplied prompt session, and state/debug
|
dependencies, input bytes, a supplied prompt session, and state/debug
|
||||||
collaborators. It returns logical output files, normalized artifacts, recorded
|
collaborators. It returns logical output files, normalized artifacts, recorded
|
||||||
rejections and warnings, manifest provenance, and checkpoint decisions. The
|
rejections, grouped diagnostics, manifest provenance, and checkpoint decisions. The
|
||||||
CLI owns process arguments, configuration discovery, session resolution,
|
CLI owns process arguments, configuration discovery, session resolution,
|
||||||
physical roots, and placement of returned output files.
|
physical roots, and placement of returned output files.
|
||||||
|
|
||||||
@@ -32,13 +32,26 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
|
|||||||
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
||||||
requested lanes where that is supported, resolves validator chains, checks
|
requested lanes where that is supported, resolves validator chains, checks
|
||||||
module capabilities and typed artifact compatibility, validates options, and
|
module capabilities and typed artifact compatibility, validates options, and
|
||||||
assigns a deterministic resolved-composition digest. The resolved pipeline
|
assigns a deterministic resolved-composition digest. A correction protocol is
|
||||||
contains bindings and declared reference targets, not external reference bytes.
|
selected from each eligible LLM-backed producer specification and becomes part
|
||||||
|
of that resolved identity; only `single_response_v1` is currently supported.
|
||||||
|
Preparation rejects an LLM-backed producer that combines a non-empty validator
|
||||||
|
chain with positive producer retries unless it declares that protocol. Producers
|
||||||
|
without validators or without retries remain valid without correction support.
|
||||||
|
The resolved pipeline contains bindings and declared reference targets, not
|
||||||
|
external reference bytes.
|
||||||
After selection, the resolver applies command, binding, and pipeline profile
|
After selection, the resolver applies command, binding, and pipeline profile
|
||||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||||
an empty resolved binding profile. Deterministic bindings remain profile-free.
|
an empty resolved binding profile. It resolves structural output repair
|
||||||
These effective values are part of the digest, so execution and checkpoint
|
separately: a binding's `structured_output_repair_attempts` value wins, then a
|
||||||
consumers do not repeat profile inheritance.
|
pipeline value applies to LLM-backed bindings and validators, and omission
|
||||||
|
leaves the prompt-owned policy intact. An explicit repair value on a
|
||||||
|
deterministic binding is rejected. Resolved bindings own copied repair values,
|
||||||
|
and these effective values are part of the digest, so execution and checkpoint
|
||||||
|
consumers do not repeat profile inheritance or configuration resolution.
|
||||||
|
Each LLM request receives its own copy of that resolved value. PromptKit spends
|
||||||
|
it only for structural correction inside one completion; the runner's binding
|
||||||
|
retry policy remains the separate outer budget for complete stage attempts.
|
||||||
Configuration resolution supplies the selected profile and catalog; see
|
Configuration resolution supplies the selected profile and catalog; see
|
||||||
[Configuration Internals](configuration.md).
|
[Configuration Internals](configuration.md).
|
||||||
|
|
||||||
@@ -46,16 +59,20 @@ External reference materialization happens before preparation. The materializer
|
|||||||
checks that each slot is declared by the selected module, resolves a file path
|
checks that each slot is declared by the selected module, resolves a file path
|
||||||
relative to the correct configuration or working-directory origin, reads
|
relative to the correct configuration or working-directory origin, reads
|
||||||
UTF-8 text, verifies media type and size limits, and retains bounded
|
UTF-8 text, verifies media type and size limits, and retains bounded
|
||||||
provenance. A generated-artifact selector remains declared but has no bytes
|
provenance. For a positive slot limit, it reads at most the limit plus one byte
|
||||||
until its producing step completes.
|
and rejects overflow before retaining content. A generated-artifact selector
|
||||||
|
remains declared but has no bytes until its producing step completes.
|
||||||
|
|
||||||
Preparation is the construction boundary. It validates the resolved shape and
|
Preparation is the construction boundary. It validates the resolved shape and
|
||||||
registry set, clones the resolved data, then constructs the input adapter,
|
registry set, clones the resolved data, then constructs the input adapter,
|
||||||
chunker, stage-local validators, every typed lane, and output encoder with
|
chunker, stage-local validators, every typed lane, and output encoder. The
|
||||||
cloned options, references, and shared dependencies. It also collects stable
|
prepared producer metadata preserves each selected correction protocol, and
|
||||||
checkpoint fingerprints. Missing registrations, incompatible typed entries,
|
the resolved digest carrying that metadata participates in checkpoint identity.
|
||||||
nil implementations, and constructor failures are reported before source
|
Each registered builder receives its own cloned build request immediately
|
||||||
parsing or any stage operation begins.
|
before its module-owned code runs. Preparation also collects stable checkpoint
|
||||||
|
fingerprints. Missing registrations, incompatible typed entries, nil
|
||||||
|
implementations, and constructor failures are reported before source parsing
|
||||||
|
or any stage operation begins.
|
||||||
|
|
||||||
An output encoder can opt into source-evidence publication through its output
|
An output encoder can opt into source-evidence publication through its output
|
||||||
policy. Preparation keeps the configured lane allowlist and active lanes
|
policy. Preparation keeps the configured lane allowlist and active lanes
|
||||||
@@ -115,18 +132,52 @@ for started workers, and prevents output encoding.
|
|||||||
|
|
||||||
Every chunk, extract, merge, and normalize candidate passes its resolved
|
Every chunk, extract, merge, and normalize candidate passes its resolved
|
||||||
validator chain. Validators receive immutable canonical input appropriate to
|
validator chain. Validators receive immutable canonical input appropriate to
|
||||||
their target: chunks, typed values, or serialized codec bytes. They may
|
their target: chunks, codec-decoded typed candidates, or serialized codec
|
||||||
approve, approve with warnings, reject, or fail. A rejection is an ordinary
|
bytes. Each typed validator receives a newly decoded value from the one
|
||||||
pipeline result; a validator error is a framework error.
|
candidate serialization for that attempt, while serialized validators receive
|
||||||
|
separately owned representation bytes and schema metadata. They may approve,
|
||||||
|
approve with warnings, reject, fail, or be skipped when a runtime prerequisite
|
||||||
|
is unavailable. The shared executor settles every configured validator in
|
||||||
|
order. A failed LLM-backed validator retries only itself against the same
|
||||||
|
immutable candidate; it does not regenerate the producer or alter the
|
||||||
|
validator request. Rejections stop that validator, while other configured
|
||||||
|
validators still run. The executor retains ordered results, bounded
|
||||||
|
deduplicated correction guidance from rejections, and only the final exhausted
|
||||||
|
failure outcome for each validator. The correction builder keeps first
|
||||||
|
occurrence order, omits internal reason codes, validator names, and operator
|
||||||
|
messages, and requests one complete replacement. Missing guidance or an
|
||||||
|
oversized aggregate is a framework contract error; guidance is never inferred
|
||||||
|
or truncated.
|
||||||
|
|
||||||
The runner applies the binding's retry policy around a stage operation and its
|
The runner applies the binding's retry policy around a stage operation and its
|
||||||
complete validation chain. It preserves warnings only from the final accepted
|
complete validation chain. It preserves terminal diagnostics only from the final accepted
|
||||||
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
|
or rejected attempt, plus one fixed validation-incomplete warning per validator whose execution
|
||||||
directives consume this same budget and validate any final safe fallback through
|
budget was exhausted under `warn_continue`. Cancellation stops retries.
|
||||||
the normalizer chain.
|
Normalizer-specific retry directives consume this same budget and validate any
|
||||||
|
final safe fallback through the normalizer chain.
|
||||||
|
|
||||||
|
The artifact-neutral producer-attempt state machine owns that shared budget,
|
||||||
|
attempt provenance, semantic-correction material, and terminal-policy
|
||||||
|
selection. It accepts producer and complete-validation closures, so artifact
|
||||||
|
materialization, cache handling, checkpoints, and debug output stay at the
|
||||||
|
operation boundary. It distinguishes operational, structural, module-requested,
|
||||||
|
and semantic retries. A semantic retry is available only for a valid latest
|
||||||
|
`single_response_v1` candidate; a deterministic or no-model rejection instead
|
||||||
|
settles the semantic policy immediately. Structural-output errors alone use the
|
||||||
|
structural policy, and validation failure without rejection settles the
|
||||||
|
validator-failure policy without regenerating the producer.
|
||||||
|
|
||||||
|
Chunk planning uses this state machine for generated plans. A rejected or
|
||||||
|
validation-incomplete automatic cache hit is not model material and therefore
|
||||||
|
falls through to a fresh initial generation at producer attempt one; it neither
|
||||||
|
receives a correction, consumes retry budget, promotes cached-candidate
|
||||||
|
diagnostics, nor overwrites the stored record. An incomplete cache validation
|
||||||
|
under `fail_run` terminates instead. Only a newly generated, completely
|
||||||
|
validated plan is published to the chunk-plan store. Rejected plans never
|
||||||
|
advance, and validation-incomplete plans remain unpublishable.
|
||||||
|
|
||||||
After terminal lane work, the runner assembles manifest provenance, normalized
|
After terminal lane work, the runner assembles manifest provenance, normalized
|
||||||
artifacts, rejections, warnings, and an optional accepted chunk map. When an
|
artifacts, rejections, final grouped diagnostics, and an optional accepted chunk map. When an
|
||||||
output policy selected evidence lanes, it decodes accepted serialized normalize
|
output policy selected evidence lanes, it decodes accepted serialized normalize
|
||||||
outputs through their registered codecs and invokes the prepared typed
|
outputs through their registered codecs and invokes the prepared typed
|
||||||
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
|
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
|
||||||
@@ -137,6 +188,15 @@ The CLI publishes those files only after the runner returns without a framework
|
|||||||
error. Logical file names and schemas are defined by the [output integration
|
error. Logical file names and schemas are defined by the [output integration
|
||||||
contracts](../integrations/).
|
contracts](../integrations/).
|
||||||
|
|
||||||
|
For every completed producer disposition, the runner projects one bounded
|
||||||
|
validation summary to the manifest, the affected rejection when present, and
|
||||||
|
the CLI result receipt. The summary records status, configured-order rejecting
|
||||||
|
validators and reason codes, incomplete validators, producer-attempt count,
|
||||||
|
and terminal action. It contains no operator message, correction guidance, or
|
||||||
|
model response. `complete`, `rejected`, and `incomplete` describe the final
|
||||||
|
candidate disposition; a run-level `incomplete` status indicates at least one
|
||||||
|
current-run output advanced under `warn_continue`.
|
||||||
|
|
||||||
## Checkpoint And Debug Hooks
|
## Checkpoint And Debug Hooks
|
||||||
|
|
||||||
The runner receives checkpoint and debug interfaces rather than roots. It
|
The runner receives checkpoint and debug interfaces rather than roots. It
|
||||||
@@ -146,6 +206,20 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
|
|||||||
Selective recomputation can require a canonical accepted normalized predecessor
|
Selective recomputation can require a canonical accepted normalized predecessor
|
||||||
before a dependent lane starts.
|
before a dependent lane starts.
|
||||||
|
|
||||||
|
The runner writes successful checkpoint artifacts only after complete accepted
|
||||||
|
validation. Chunk plans follow the same rule for publication. A rejection,
|
||||||
|
invalid structured response, or incomplete validation is never reusable state;
|
||||||
|
the current run may still hand off an otherwise valid `warn_continue` result
|
||||||
|
according to its terminal policy. The runner carries private reuse eligibility
|
||||||
|
through extract, merge, normalize, and generated-reference handoff. Any stage
|
||||||
|
derived from incomplete validation skips both checkpoint lookup and all
|
||||||
|
checkpoint publication even when that stage's own validation completes.
|
||||||
|
External references and fully validated generated references remain eligible.
|
||||||
|
Attempt debug records retain safe kind,
|
||||||
|
validator, repair-usage, policy, and terminal-decision provenance. Full
|
||||||
|
assistant and correction content remains confined to the requested detailed
|
||||||
|
LLM trace.
|
||||||
|
|
||||||
Debug recording is attempt-scoped and application-owned. A failure to persist
|
Debug recording is attempt-scoped and application-owned. A failure to persist
|
||||||
required debug data is a framework error. State roots, persistence, reason-code
|
required debug data is a framework error. State roots, persistence, reason-code
|
||||||
meanings, resume, and cleanup are intentionally owned by
|
meanings, resume, and cleanup are intentionally owned by
|
||||||
|
|||||||
@@ -47,6 +47,9 @@ The serialized
|
|||||||
they do not describe a current public state surface.
|
they do not describe a current public state surface.
|
||||||
|
|
||||||
Ordered-step lane checkpoints include the step identity in their storage scope.
|
Ordered-step lane checkpoints include the step identity in their storage scope.
|
||||||
|
Accepted step and lane identities are encoded injectively before becoming
|
||||||
|
filesystem path components, while ordinary safe identifiers retain their
|
||||||
|
readable paths.
|
||||||
When a later lane consumes a generated artifact, its dependency fingerprints
|
When a later lane consumes a generated artifact, its dependency fingerprints
|
||||||
include the producer's artifact kind, complete schema identity, media type,
|
include the producer's artifact kind, complete schema identity, media type,
|
||||||
canonical content digest, and size. Ordinary resume compares those fingerprints
|
canonical content digest, and size. Ordinary resume compares those fingerprints
|
||||||
@@ -61,12 +64,13 @@ Ordinary resume loads extract, merge, and normalize checkpoints progressively
|
|||||||
and may execute later lane stages after an earlier cache miss. Selective
|
and may execute later lane stages after an earlier cache miss. Selective
|
||||||
recomputation instead asks the loader for the required producer's accepted
|
recomputation instead asks the loader for the required producer's accepted
|
||||||
normalize artifact. That lookup reuses the existing normalize files, requires
|
normalize artifact. That lookup reuses the existing normalize files, requires
|
||||||
workspace schema v3 plus an exact non-empty invocation identity, and deliberately
|
workspace schema v4 plus an exact non-empty invocation identity, and deliberately
|
||||||
does not require extract or merge checkpoint files or dependency fingerprints.
|
does not require extract or merge checkpoint files or dependency fingerprints.
|
||||||
The runner performs canonical codec and producer-provenance validation before
|
The runner performs canonical codec and producer-provenance validation before
|
||||||
cloning the artifact into normal step output. Success restores only stored
|
cloning the artifact into normal step output. Success restores only stored
|
||||||
normalize warnings and emits one normalize decision; failure retains the files,
|
normalize diagnostics and emits one normalize decision; failure retains the
|
||||||
records the decision, and stops without executing the producer or consumer.
|
files, records the decision, and stops without executing the producer or
|
||||||
|
consumer.
|
||||||
|
|
||||||
The loader assigns a typed category and reason code at each validation site;
|
The loader assigns a typed category and reason code at each validation site;
|
||||||
diagnostic prose is not classified after the fact. The runner then applies
|
diagnostic prose is not classified after the fact. The runner then applies
|
||||||
@@ -94,7 +98,7 @@ owns the operator workflow and stable reason-code meanings.
|
|||||||
|
|
||||||
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
|
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
|
||||||
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
|
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
|
||||||
resolution, run, warning, and failure artifacts. `internal/framework/debug`
|
resolution, run, final grouped diagnostic, and failure artifacts. `internal/framework/debug`
|
||||||
implements the pipeline-facing trace recorder under the trace root.
|
implements the pipeline-facing trace recorder under the trace root.
|
||||||
|
|
||||||
The CLI allocates a bundle before pipeline resolution and treats requested
|
The CLI allocates a bundle before pipeline resolution and treats requested
|
||||||
|
|||||||
@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
|
|||||||
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
||||||
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
||||||
|
|
||||||
|
## Source Deployment
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. Install a pinned source release
|
||||||
|
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Pin the exact tag in deployment automation rather than following a branch.
|
||||||
|
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
|
||||||
|
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
|
||||||
|
The maintainer publication process, including tag guards and verification,
|
||||||
|
belongs to [Source Releases](release.md). macOS builds are best-effort for
|
||||||
|
development, and Windows is unsupported.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
Each run can use independent roots with different retention and access-control
|
Each run can use independent roots with different retention and access-control
|
||||||
@@ -72,6 +89,9 @@ provider call or credentials:
|
|||||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
|
An unset optional `api_key_env` reaches the provider without authorization and
|
||||||
|
may receive a 401 or 403 response.
|
||||||
|
|
||||||
Profile paths are currently resolved from the process working directory, not
|
Profile paths are currently resolved from the process working directory, not
|
||||||
from the configuration file. The complete example's
|
from the configuration file. The complete example's
|
||||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||||
@@ -85,10 +105,40 @@ and resolves configuration before module preparation and source parsing. It
|
|||||||
then performs any permitted cache lookup, executes the pipeline, and publishes
|
then performs any permitted cache lookup, executes the pipeline, and publishes
|
||||||
logical output files only after a successful runner result.
|
logical output files only after a successful runner result.
|
||||||
|
|
||||||
On success, the command reports the output bundle path. A warning-bearing run
|
On success, the command reports the output bundle path. A run with actionable
|
||||||
still succeeds and reports its warning count on standard error. Errors and
|
process warnings still succeeds and reports warning-group and occurrence counts
|
||||||
|
on standard error; advisory and observation findings do not produce a warning
|
||||||
|
line. Errors and
|
||||||
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
||||||
|
|
||||||
|
## Validation Retries And Terminal Outcomes
|
||||||
|
|
||||||
|
Each producer binding has one outer **retries** budget. It covers complete
|
||||||
|
producer attempts for operational failures, invalid structured output,
|
||||||
|
normalizer fallback retry, and semantic correction. It is independent from
|
||||||
|
PromptKit's structural-repair calls inside one completion and from an
|
||||||
|
LLM-backed validator's own retry budget. A semantic correction rebuilds the
|
||||||
|
ordinary producer request and supplies only the latest rejected model response
|
||||||
|
plus aggregated validator guidance; it is not a conversation replay.
|
||||||
|
|
||||||
|
After the applicable budgets are exhausted, the resolved
|
||||||
|
[`validation_policy`](config.md#pipelines) determines the result. Structural
|
||||||
|
failure and semantic rejection normally fail the run; an explicit
|
||||||
|
`reject_output` records a rejection and allows unrelated work to finish. A
|
||||||
|
validator execution failure normally uses `warn_continue`, which keeps an
|
||||||
|
otherwise accepted result in the current run with `incomplete` validation
|
||||||
|
provenance. It emits one bounded warning for every validator whose execution
|
||||||
|
budget was exhausted. A corrected result that later passes validation does not
|
||||||
|
retain diagnostics from abandoned attempts.
|
||||||
|
|
||||||
|
Treat a successful process exit as a completed run, not as proof that every
|
||||||
|
candidate was fully validated. Inspect the receipt's `validation_status`,
|
||||||
|
`validation_summaries`, rejection count, and warning group and occurrence
|
||||||
|
counts when an orchestrator requires complete validation. The durable fields
|
||||||
|
and their meanings are owned by the
|
||||||
|
[run-result receipt](integrations/run-result.md) and
|
||||||
|
[published JSON output contract](integrations/json-output.md).
|
||||||
|
|
||||||
## Output Bundles
|
## Output Bundles
|
||||||
|
|
||||||
Each successful run receives a generated safe run identifier and writes beneath:
|
Each successful run receives a generated safe run identifier and writes beneath:
|
||||||
@@ -110,9 +160,9 @@ are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
|
|||||||
[evidence context](integrations/evidence-context.md) contains source-unit text
|
[evidence context](integrations/evidence-context.md) contains source-unit text
|
||||||
and metadata. It is not a cache or debug artifact: retain it with the output
|
and metadata. It is not a cache or debug artifact: retain it with the output
|
||||||
bundle only for as long as consumers need it, and apply source-content access
|
bundle only for as long as consumers need it, and apply source-content access
|
||||||
controls to the entire bundle. Selected lanes may collectively cite most of a
|
controls to the entire bundle. Its selected source-unit excerpt may include
|
||||||
transcript, so a broad allowlist can make the evidence artifact nearly as
|
every source unit once when coverage is broad or its configured window is
|
||||||
sensitive and large as the source itself.
|
large, so do not assume a byte or token reduction or reduced sensitivity.
|
||||||
|
|
||||||
## Chunk-Plan Cache
|
## Chunk-Plan Cache
|
||||||
|
|
||||||
@@ -138,7 +188,9 @@ The configured cache mode controls one invocation:
|
|||||||
A reused plan is still materialized and validated against the current source.
|
A reused plan is still materialized and validated against the current source.
|
||||||
If a prior plan no longer gives acceptable results, use a refresh run rather
|
If a prior plan no longer gives acceptable results, use a refresh run rather
|
||||||
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
||||||
chunking work.
|
chunking work. A plan accepted only under incomplete validation is not
|
||||||
|
published, and a rejected cache hit falls through to ordinary generation rather
|
||||||
|
than becoming a correction candidate.
|
||||||
|
|
||||||
## Checkpoint Recording, Resume, And Recompute
|
## Checkpoint Recording, Resume, And Recompute
|
||||||
|
|
||||||
@@ -163,6 +215,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
|
|||||||
runtime identities, so checkpoints created under one state are not reused by
|
runtime identities, so checkpoints created under one state are not reused by
|
||||||
either of the others.
|
either of the others.
|
||||||
|
|
||||||
|
Only accepted, completely validated chunk, extract, merge, and normalize
|
||||||
|
results are checkpointed for reuse. Rejected, structurally invalid, and
|
||||||
|
validation-incomplete producer results remain non-reusable, even when a
|
||||||
|
`warn_continue` result advanced during its original run. A resumed invocation
|
||||||
|
therefore reruns that producer rather than treating degraded state as accepted.
|
||||||
|
|
||||||
Checkpoint state is confined below an identity-specific path:
|
Checkpoint state is confined below an identity-specific path:
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
@@ -217,13 +275,16 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
|
|||||||
~~~
|
~~~
|
||||||
|
|
||||||
The summary contains redacted invocation and resolution information plus run,
|
The summary contains redacted invocation and resolution information plus run,
|
||||||
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
|
final grouped diagnostic, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
|
||||||
contains allowlisted application diagnostic records and can include source or
|
terminal records contain bounded attempt kinds, validator outcomes, policy,
|
||||||
derived application data. Neither surface is a cache input. Do not treat a
|
decision, PromptKit repair count, and usage; they do not contain assistant
|
||||||
debug bundle as safe to share merely because its configuration summary is
|
responses or complete correction messages. The trace contains allowlisted
|
||||||
redacted. Invocation metadata omits reasoning effort when it is inherited,
|
application diagnostic records and can include source, model, and correction
|
||||||
records the replacement value when one is supplied, and records an empty value
|
content. Neither surface is a cache input. Do not treat a debug bundle as safe
|
||||||
when inherited reasoning was explicitly cleared.
|
to share merely because its configuration summary is redacted. Invocation
|
||||||
|
metadata omits reasoning effort when it is inherited, records the replacement
|
||||||
|
value when one is supplied, and records an empty value when inherited reasoning
|
||||||
|
was explicitly cleared.
|
||||||
|
|
||||||
Notarius never creates debug state without an explicit request and never
|
Notarius never creates debug state without an explicit request and never
|
||||||
automatically deletes a requested bundle. If allocation succeeds, the command
|
automatically deletes a requested bundle. If allocation succeeds, the command
|
||||||
@@ -255,14 +316,29 @@ Provider execution settings and the generation timeout come from the selected
|
|||||||
PromptKit profile. The invocation-only **--reasoning-effort** and
|
PromptKit profile. The invocation-only **--reasoning-effort** and
|
||||||
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
||||||
for all LLM-backed calls in one run without changing the profile. PromptKit
|
for all LLM-backed calls in one run without changing the profile. PromptKit
|
||||||
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
|
structural output repair happens within one structured-completion call. Its
|
||||||
complete module operation and validation chain as defined by
|
effective `structured_output_repair_attempts` limit is resolved from the
|
||||||
[module bindings](config.md#module-bindings-and-validators).
|
selected binding, then the pipeline, then the prompt declaration; see
|
||||||
|
[module bindings](config.md#module-bindings-and-validators). This is distinct
|
||||||
|
from Notarius binding **retries**, which rerun the complete module operation
|
||||||
|
and validation chain and do not consume or replenish the structural-repair
|
||||||
|
limit. The maintained production prompts declare one repair attempt, paid only
|
||||||
|
after a structural failure. One structured completion with repair budget **R**
|
||||||
|
makes at most **R + 1** serial provider calls. If one stage attempt performs
|
||||||
|
**C** structured completions, a binding with **retries: N** has a maximum of
|
||||||
|
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
|
||||||
|
corresponding invocation counts and budgets. This is an upper bound, not a
|
||||||
|
promise that every call reaches a provider.
|
||||||
|
|
||||||
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
||||||
effective generation timeout adds an inner request deadline, while zero
|
effective generation timeout adds an inner request deadline, while zero
|
||||||
disables only that generation deadline. The HTTP client timeout remains a
|
disables only that generation deadline. The HTTP client timeout remains a
|
||||||
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
||||||
|
Repairs are serial within the same caller context, so their worst-case latency
|
||||||
|
and cost follow the provider-call bound above; provision run deadlines and
|
||||||
|
provider budgets accordingly. Credentials remain optional unless the selected
|
||||||
|
PromptKit profile requires one, in which case preparation fails before a
|
||||||
|
provider call when its configured credential is unavailable.
|
||||||
The pinned upstream boundary and profile-format links are in
|
The pinned upstream boundary and profile-format links are in
|
||||||
[PromptKit Integration](integrations/pkg-promptkit.md).
|
[PromptKit Integration](integrations/pkg-promptkit.md).
|
||||||
|
|
||||||
@@ -279,6 +355,11 @@ positive value makes the effective active local-generation bound the smaller
|
|||||||
of **total_llm** and that local limit, so a local limit of four permits no more
|
of **total_llm** and that local limit, so a local limit of four permits no more
|
||||||
than four active local generations.
|
than four active local generations.
|
||||||
|
|
||||||
|
The Notarius scheduler admits one logical structured completion and holds that
|
||||||
|
permit while PromptKit performs its serial corrective calls. PromptKit applies
|
||||||
|
its selected-backend admission to each provider call; Notarius does not
|
||||||
|
reacquire a permit or add another scheduler for a repair.
|
||||||
|
|
||||||
For a positive local limit, PromptKit owns its default waiting capacity and
|
For a positive local limit, PromptKit owns its default waiting capacity and
|
||||||
admission behavior. When a PromptKit backend has admitted all active and queued
|
admission behavior. When a PromptKit backend has admitted all active and queued
|
||||||
work, a new call fails as capacity exhaustion before generation. The adapter
|
work, a new call fails as capacity exhaustion before generation. The adapter
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ DAGs or a general workflow language. Every stage remains explicit; general
|
|||||||
chunking, merging, or normalization behavior must not be hidden inside an
|
chunking, merging, or normalization behavior must not be hidden inside an
|
||||||
extractor.
|
extractor.
|
||||||
|
|
||||||
|
A stage module is one configured implementation of one pipeline stage. An
|
||||||
|
artifact family is the cohesive domain feature that owns an artifact across
|
||||||
|
the explicit stages and supporting codecs, validators, prompts, identity
|
||||||
|
rules, and reference projections. Artifact-family ownership does not combine
|
||||||
|
stages or alter the fixed pipeline.
|
||||||
|
|
||||||
Input and chunking are pipeline-wide. Each selected artifact lane owns its
|
Input and chunking are pipeline-wide. Each selected artifact lane owns its
|
||||||
extract, merge, and normalize stages, and the output stage aggregates the run's
|
extract, merge, and normalize stages, and the output stage aggregates the run's
|
||||||
lane outcomes.
|
lane outcomes.
|
||||||
@@ -39,6 +45,12 @@ implementations. Domain-neutral model and framework layers provide reusable
|
|||||||
policy, contracts, and orchestration. Concrete input, pipeline, output, and
|
policy, contracts, and orchestration. Concrete input, pipeline, output, and
|
||||||
validation extensions depend inward on those generic layers.
|
validation extensions depend inward on those generic layers.
|
||||||
|
|
||||||
|
Semantic reconciliation is one such domain-neutral framework mechanism. It
|
||||||
|
prepares bounded source context, invokes a shared model-judgment protocol,
|
||||||
|
validates proposals, and applies safe plans through typed policies supplied by
|
||||||
|
the consuming artifact family. It does not own domain identity, durable IDs,
|
||||||
|
warning semantics, or artifact construction rules.
|
||||||
|
|
||||||
Generic layers must not depend on production extensions. Concrete extensions
|
Generic layers must not depend on production extensions. Concrete extensions
|
||||||
must not compose the application or take ownership of process behavior. The
|
must not compose the application or take ownership of process behavior. The
|
||||||
current packages implementing these layers are inventoried in
|
current packages implementing these layers are inventoried in
|
||||||
@@ -134,8 +146,9 @@ lanes, validators, and LLM profile: the canonical source digest selects the
|
|||||||
plan, while the current run still applies its configured chunk validators to
|
plan, while the current run still applies its configured chunk validators to
|
||||||
the materialized chunks.
|
the materialized chunks.
|
||||||
|
|
||||||
The framework owns orchestration and handoff provenance. Modules return logical
|
The framework owns orchestration, origin enrichment, aggregation, and handoff
|
||||||
results and warnings; they do not own CLI reporting, physical output, cache, or
|
provenance. Modules return logical results and classified diagnostics; they do
|
||||||
|
not own CLI reporting, physical output, cache, or
|
||||||
debug roots, durable file placement, or checkpoint and debug lifecycle.
|
debug roots, durable file placement, or checkpoint and debug lifecycle.
|
||||||
|
|
||||||
After pipeline-wide chunking, extraction uses bounded framework concurrency.
|
After pipeline-wide chunking, extraction uses bounded framework concurrency.
|
||||||
@@ -147,26 +160,46 @@ may overlap. The framework must not create unbounded goroutines per lane or
|
|||||||
chunk.
|
chunk.
|
||||||
|
|
||||||
Completion timing does not choose public ordering or errors. The coordinator
|
Completion timing does not choose public ordering or errors. The coordinator
|
||||||
orders accepted artifacts, warnings, rejections, checkpoint events, and
|
orders accepted artifacts, grouped diagnostics, rejections, checkpoint events, and
|
||||||
framework errors by stable pipeline scope. Rejections do not cancel unrelated
|
framework errors by stable pipeline scope. Rejections do not cancel unrelated
|
||||||
work. A framework error cancels derived work, prevents undispatched work from
|
work. A framework error cancels derived work, prevents undispatched work from
|
||||||
starting, waits for started work, and prevents output encoding.
|
starting, waits for started work, and prevents output encoding.
|
||||||
|
|
||||||
|
Warnings are process-only signals: configuration degradation, approved fallback,
|
||||||
|
or incomplete configured validation. Quality uncertainty and grounding findings
|
||||||
|
are advisories; successful canonicalization and cleanup are observations.
|
||||||
|
Modules choose that semantic classification, while the framework attaches
|
||||||
|
origin, aggregates groups, enforces bounds, and presents final collections.
|
||||||
|
An ordinary successful run therefore has zero warnings. See
|
||||||
|
[ADR-0015](../adr/0015-separate-process-warnings-from-quality-diagnostics.md)
|
||||||
|
for the decision rationale.
|
||||||
|
|
||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
Validation is a framework-managed boundary around outputs from chunk, extract,
|
Validation is a framework-managed boundary around outputs from chunk, extract,
|
||||||
merge, and normalize stages. Validators receive immutable stage output
|
merge, and normalize stages. Validators receive immutable stage output and
|
||||||
and make an explicit whole-output decision: approve, approve with warnings, or
|
make an explicit whole-output decision: approve, reject, fail, or skip when a
|
||||||
reject.
|
runtime prerequisite is unavailable.
|
||||||
|
|
||||||
Typed artifact validators receive the domain value directly. Chunk validators
|
Typed artifact validators receive the domain value directly. Chunk validators
|
||||||
receive source-zone chunks, while serialized validators receive immutable
|
receive source-zone chunks, while serialized validators receive immutable
|
||||||
representation bytes and declared schema metadata. A validator registered for
|
representation bytes and declared schema metadata. A validator registered for
|
||||||
one target or artifact kind cannot satisfy an incompatible selection.
|
one target or artifact kind cannot satisfy an incompatible selection.
|
||||||
|
|
||||||
Rejection is a recorded pipeline outcome, not a framework execution error.
|
The framework runs every applicable validator sequentially in configured order.
|
||||||
Validator execution failures are framework errors. Rejected output does not
|
It aggregates rejections, exhausted validator failures, and skips before the
|
||||||
advance to the next stage.
|
producer policy chooses a disposition. A completed rejection never advances.
|
||||||
|
With no rejection, an exhausted validator failure may fail the run or, under
|
||||||
|
the configured `warn_continue` policy, advance a structurally valid candidate
|
||||||
|
with explicit incomplete-validation provenance. Validators report findings;
|
||||||
|
they do not choose candidate disposition.
|
||||||
|
|
||||||
|
A completed rejection supplies a stable reason code for internal provenance
|
||||||
|
and bounded actionable correction guidance for the candidate producer. Reason
|
||||||
|
codes, validator keys, and operator-facing messages remain diagnostic data;
|
||||||
|
they are not model instructions. The framework constructs model-facing retry
|
||||||
|
text only from the semantic guidance and fails the contract rather than
|
||||||
|
inventing or truncating missing guidance.
|
||||||
|
|
||||||
Default validator chains are production composition policy and are registered
|
Default validator chains are production composition policy and are registered
|
||||||
centrally by stage and module. Configuration may replace a stage-local default,
|
centrally by stage and module. Configuration may replace a stage-local default,
|
||||||
@@ -183,11 +216,29 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
|
|||||||
and interpretation of structured output. Provider adapters do not own source-
|
and interpretation of structured output. Provider adapters do not own source-
|
||||||
or domain-specific prompt logic.
|
or domain-specific prompt logic.
|
||||||
|
|
||||||
|
PromptKit owns bounded structural correction within one structured completion.
|
||||||
|
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
|
||||||
|
the two budgets must remain separate.
|
||||||
|
|
||||||
|
An LLM-backed producer can participate in semantic correction only when it
|
||||||
|
declares `single_response_v1` and returns the exact one response that directly
|
||||||
|
controlled its candidate. On an actionable rejection, the framework rebuilds
|
||||||
|
the ordinary request and appends only the latest defective response as an
|
||||||
|
`assistant` message plus one aggregated `user` correction message. This is a
|
||||||
|
fresh replacement request, not a growing conversation. The retry budgets,
|
||||||
|
terminal policy, and sensitive-data rationale are recorded in
|
||||||
|
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md).
|
||||||
|
|
||||||
When a model selects an application entity, callers must supply a contextual
|
When a model selects an application entity, callers must supply a contextual
|
||||||
selection and deterministically attach the opaque application identity whenever
|
selection and deterministically attach the opaque application identity whenever
|
||||||
the selection resolves exactly. Models do not receive or reproduce opaque
|
the selection resolves exactly. Models do not receive or reproduce opaque
|
||||||
application identifiers; [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
|
application identifiers. Semantic reconciliation may instead expose
|
||||||
records the rationale and limited request-local-label exception.
|
contiguous, one-based candidate handles that exist only for one request;
|
||||||
|
deterministic code resolves them before typed application, and they never
|
||||||
|
become durable identity. This is the approved request-local-label application
|
||||||
|
of [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
|
||||||
|
recorded by
|
||||||
|
[ADR-0013](../adr/0013-use-request-local-candidate-handles-for-semantic-reconciliation.md).
|
||||||
|
|
||||||
LLM calls and other external operations accept cancellation and respect
|
LLM calls and other external operations accept cancellation and respect
|
||||||
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
|
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
|
||||||
@@ -212,7 +263,8 @@ invalid or incompatible.
|
|||||||
|
|
||||||
Run manifests record enough resolved pipeline, module, source, reference, and
|
Run manifests record enough resolved pipeline, module, source, reference, and
|
||||||
LLM provenance to make a run auditable after configuration changes. Manifests
|
LLM provenance to make a run auditable after configuration changes. Manifests
|
||||||
record identities and summaries rather than secret or large payload content.
|
record identities and bounded validation summaries rather than secret, raw
|
||||||
|
model, correction, or large payload content.
|
||||||
|
|
||||||
## State, Output, And Safety
|
## State, Output, And Safety
|
||||||
|
|
||||||
@@ -232,18 +284,39 @@ an invocation that explicitly requests resume. Debug is never a cache input and
|
|||||||
is never created without an explicit request. Pipeline modules receive
|
is never created without an explicit request. Pipeline modules receive
|
||||||
collaborator interfaces and never physical roots.
|
collaborator interfaces and never physical roots.
|
||||||
|
|
||||||
|
Only accepted, completely validated producer output is reusable checkpoint or
|
||||||
|
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
|
||||||
|
results cannot become cache or checkpoint inputs, even when a
|
||||||
|
`warn_continue` result is allowed to advance in the current run. This
|
||||||
|
ineligibility follows derived merge and normalize results and generated
|
||||||
|
references for the remainder of the run: current-run handoff remains allowed,
|
||||||
|
but no dependent cache or checkpoint may be loaded or published.
|
||||||
|
|
||||||
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
||||||
deletion must be narrow and explicit. Notarius never automatically deletes
|
deletion must be narrow and explicit. Notarius never automatically deletes
|
||||||
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
||||||
|
|
||||||
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
||||||
traces, manifests, documentation, examples, or redacted configuration. Debug
|
traces, manifests, documentation, examples, or redacted configuration. Raw
|
||||||
|
assistant responses and complete correction messages are attempt-local and are
|
||||||
|
excluded from ordinary durable records and summaries; the requested detailed
|
||||||
|
debug trace is the sole diagnostic surface allowed to retain them. Debug
|
||||||
collection is allowlisted to application-owned payloads and must not capture
|
collection is allowlisted to application-owned payloads and must not capture
|
||||||
unrelated process environment values or filesystem content. Trace data may
|
unrelated process environment values or filesystem content. Trace data may
|
||||||
contain application data and therefore inherits its sensitivity; operators own
|
contain application data and therefore inherits its sensitivity; operators own
|
||||||
access controls and retention. Physical layout and operation are defined in
|
access controls and retention. Physical layout and operation are defined in
|
||||||
[Operations](../operations.md).
|
[Operations](../operations.md).
|
||||||
|
|
||||||
|
## Platform And Distribution
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. macOS is supported only as a
|
||||||
|
best-effort development and compilation environment, while Windows is
|
||||||
|
unsupported. Notarius distributes source releases only: an immutable source
|
||||||
|
tag and its checked-in release note identify a release. The project does not
|
||||||
|
publish executable binaries, archives, installers, container images,
|
||||||
|
checksums, signatures, or package-manager entries. Maintainer release commands
|
||||||
|
and tag guards belong to [Source Releases](../release.md).
|
||||||
|
|
||||||
## Architectural Non-Goals
|
## Architectural Non-Goals
|
||||||
|
|
||||||
Notarius does not aim to provide:
|
Notarius does not aim to provide:
|
||||||
|
|||||||
@@ -65,6 +65,8 @@ secret values.
|
|||||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
|
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
|
||||||
|
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
|
|||||||
crosses these topics, choose the document that owns the task and link to the
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
other contracts.
|
other contracts.
|
||||||
|
|
||||||
|
### Releases
|
||||||
|
|
||||||
|
`docs/release.md` owns the source-release procedure. Release notes are
|
||||||
|
historical summaries, not current-state contract owners: the checked-in note at
|
||||||
|
an immutable tag records that release, while current canonical documentation
|
||||||
|
must change with the behavior it describes. Do not use a release note to defer
|
||||||
|
or replace current documentation updates.
|
||||||
|
|
||||||
### Contracts And Implementation
|
### Contracts And Implementation
|
||||||
|
|
||||||
Integration and API documents define externally observable shapes and
|
Integration and API documents define externally observable shapes and
|
||||||
|
|||||||
164
docs/release.md
Normal file
164
docs/release.md
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
# Source Releases
|
||||||
|
|
||||||
|
This procedure is for maintainers publishing Notarius source releases. A
|
||||||
|
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
|
||||||
|
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
|
||||||
|
candidate after publication; it does not publish or repair a release.
|
||||||
|
|
||||||
|
Notarius publishes no binaries, archives, checksums, signatures, containers,
|
||||||
|
package-manager entries, or Gitea release objects. Windows is not supported.
|
||||||
|
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
|
||||||
|
`v0.3.0` tags.
|
||||||
|
|
||||||
|
## Select And Describe The Release
|
||||||
|
|
||||||
|
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
|
||||||
|
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
|
||||||
|
documented CLI, configuration, durable artifact, integration, or operating
|
||||||
|
contract when its note explains the impact and required operator action. A
|
||||||
|
patch release must not intentionally break those documented contracts within
|
||||||
|
its minor line.
|
||||||
|
|
||||||
|
Create the version-matched note as part of the candidate. Every new note uses
|
||||||
|
this structure, with concise, truthful content in each section:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# Notarius vMAJOR.MINOR.PATCH
|
||||||
|
|
||||||
|
This release ...
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
```
|
||||||
|
|
||||||
|
The note is a historical summary. Link to current canonical documentation for
|
||||||
|
exact behavior, and update that documentation in the candidate rather than
|
||||||
|
using the note as a substitute.
|
||||||
|
|
||||||
|
## Prepare The Candidate
|
||||||
|
|
||||||
|
Set the selected release version and disable Go workspace use for every
|
||||||
|
candidate command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_VERSION=vMAJOR.MINOR.PATCH
|
||||||
|
export RELEASE_VERSION GOWORK=off
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the shared source-candidate checks from the repository. They cover module
|
||||||
|
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
|
||||||
|
configuration validation, and the Linux and Darwin command-build matrix:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./scripts/check-release-source.sh "$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Before committing, manually follow every changed local Markdown link and
|
||||||
|
review the candidate for unintended files, generated output, credentials, or
|
||||||
|
other unrelated changes. Commit the release note and all affected current
|
||||||
|
documentation, then run the shared checker against that exact candidate. Push
|
||||||
|
the candidate commit to `main` only after it succeeds. Record the exact commit
|
||||||
|
only after that push:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
For private-module installation, configure standard `GOPRIVATE` matching this
|
||||||
|
module and ordinary Git authentication for the hosting service before running
|
||||||
|
the verification below. The exact authentication mechanism belongs to the
|
||||||
|
maintainer environment; never record credentials or environment dumps in a
|
||||||
|
release note, command history, or repository file.
|
||||||
|
|
||||||
|
## Guard And Publish The Tag
|
||||||
|
|
||||||
|
Fetch current remote references, then run this guard without editing the
|
||||||
|
candidate. It requires `main`, a clean worktree and index, disabled workspace
|
||||||
|
use, a stable release version, the recorded and pushed commit, a matching note,
|
||||||
|
and unused local and remote tags:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git fetch origin main --tags
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
test "$GOWORK" = off
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
|
||||||
|
test -s "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
done
|
||||||
|
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Create an explicitly lightweight tag against the guarded commit, verify its
|
||||||
|
target, and push only that tag ref:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
|
||||||
|
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
|
||||||
|
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
|
||||||
|
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Never use `git push --tags`, move a published tag, or delete a published tag.
|
||||||
|
|
||||||
|
## Verify The Published Release
|
||||||
|
|
||||||
|
Confirm that the remote tag still points at the guarded commit and that the
|
||||||
|
note is available from the tagged tree:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
|
||||||
|
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
|
||||||
|
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify a fresh source installation and its diagnostic version. The temporary
|
||||||
|
directory confines the installed command to this check:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_verification_dir=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
|
||||||
|
mkdir -p "$release_verification_dir/bin"
|
||||||
|
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
|
||||||
|
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
|
||||||
|
equivalent source verification when local installation policy requires it.
|
||||||
|
`notarius --version` is diagnostic only; downstream compatibility remains
|
||||||
|
defined by the published receipt and artifact contracts.
|
||||||
|
|
||||||
|
## Failure And Correction Policy
|
||||||
|
|
||||||
|
If candidate validation fails before publication, fix the candidate on `main`,
|
||||||
|
rerun the shared checker, and repeat the guards. An unpublished local tag may
|
||||||
|
be deleted after inspection.
|
||||||
|
|
||||||
|
If the remote tag or tag CI reveals a defect, leave the published tag intact.
|
||||||
|
Fix the defect on `main`, choose a new patch version, write a new matching
|
||||||
|
note, and repeat this procedure. Do not weaken tag immutability or add release
|
||||||
|
assets as a workaround.
|
||||||
83
docs/releases/v0.4.0.md
Normal file
83
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# Notarius v0.4.0
|
||||||
|
|
||||||
|
This release strengthens LLM reliability and validation throughout the
|
||||||
|
configured pipeline, upgrades the PromptKit integration, and establishes the
|
||||||
|
source-release and downstream-consumer workflows needed for broader D&D
|
||||||
|
pipeline integration.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Notarius now distinguishes PromptKit structural-output repair from
|
||||||
|
application-owned semantic validation retries. Producer candidates can run
|
||||||
|
through complete deterministic validator chains, receive bounded semantic
|
||||||
|
correction guidance, and retry under explicit stage policies. Final run
|
||||||
|
receipts and manifests preserve bounded validation provenance, while outputs
|
||||||
|
that advance with incomplete validation remain available to the current run
|
||||||
|
without entering reusable checkpoint state.
|
||||||
|
|
||||||
|
The release also adds a maintained complete D&D subprocess-consumer workflow,
|
||||||
|
diagnostic build versions, and the source-only release procedure used to
|
||||||
|
publish this version.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
- Configuration files must use schema version 4. Version 3 is not decoded or
|
||||||
|
rewritten; rename the top-level `scriptorium` section to `promptkit` when
|
||||||
|
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
|
||||||
|
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
|
||||||
|
format and may use its profile-inheritance support. Notarius continues to
|
||||||
|
resolve operator profiles before embedded fallbacks.
|
||||||
|
- Structural-output repair and semantic stage retries are separate bounded
|
||||||
|
mechanisms. Maintained production prompts request one structural repair by
|
||||||
|
default; explicit configuration can override the supported repair count.
|
||||||
|
- Validation policy can now fail a run, reject an output, or permit an
|
||||||
|
otherwise valid candidate to advance with incomplete-validation provenance.
|
||||||
|
The application defaults are documented in
|
||||||
|
[Configuration](../config.md#pipelines).
|
||||||
|
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
|
||||||
|
optional validation summaries plus a required validation-status field.
|
||||||
|
Consumers of this pre-release contract should follow the current
|
||||||
|
[run-result receipt](../integrations/run-result.md).
|
||||||
|
- Existing D&D artifact schema identities remain unchanged. Validation and
|
||||||
|
producer-policy changes can nevertheless cause previously accepted weak
|
||||||
|
candidates to retry, reject, or fail instead.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
|
||||||
|
to `promptkit`.
|
||||||
|
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
|
||||||
|
and ensure their credential environment variables are available at run
|
||||||
|
time.
|
||||||
|
3. Run `notarius config validate --config <path> --pipeline <id>` before the
|
||||||
|
first production invocation.
|
||||||
|
4. Review `structured_output_repair_attempts`, producer retry counts, and
|
||||||
|
`validation_policy` wherever the deployment needs behavior different from
|
||||||
|
the documented defaults.
|
||||||
|
5. Update subprocess consumers to inspect receipt `validation_status` and to
|
||||||
|
tolerate the optional bounded `validation_summaries` field. A consumer that
|
||||||
|
requires fully validated artifacts should require `approved`.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
|
||||||
|
inheritance, structured-output repair, typed error classification, and the
|
||||||
|
correction-aware completion protocol.
|
||||||
|
- Added pipeline and binding configuration for structural repair and terminal
|
||||||
|
validation policy, with strict startup validation and effective-setting
|
||||||
|
provenance.
|
||||||
|
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
|
||||||
|
semantic reconciliation producers. Retry prompts contain the exact defective
|
||||||
|
response and actionable semantic correction guidance without exposing
|
||||||
|
internal reason codes or opaque entity identifiers.
|
||||||
|
- Added complete validator-chain execution, validator retry handling, bounded
|
||||||
|
warnings, terminal dispositions, and durable validation summaries.
|
||||||
|
- Prevented validation-incomplete artifacts and all derived lineage from
|
||||||
|
loading or publishing reusable checkpoints while preserving same-run
|
||||||
|
generated-reference handoff.
|
||||||
|
- Tightened cached chunk-plan validation so only completely validated plans are
|
||||||
|
reused or replace stored plans.
|
||||||
|
- Added a machine-readable subprocess receipt workflow and complete D&D
|
||||||
|
consumer documentation covering all maintained artifacts.
|
||||||
|
- Added source-release checks, immutable lightweight-tag guidance, Linux and
|
||||||
|
Darwin build verification, and diagnostic `notarius --version` output.
|
||||||
85
docs/releases/v0.5.0.md
Normal file
85
docs/releases/v0.5.0.md
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
# Notarius v0.5.0
|
||||||
|
|
||||||
|
This release separates actionable process warnings from extraction-quality
|
||||||
|
advisories and routine normalization observations, giving operators a quiet
|
||||||
|
warning channel without discarding durable diagnostic detail.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Notarius now carries one validated, origin-aware diagnostic contract from
|
||||||
|
producers and validators through retries, reusable state, output publication,
|
||||||
|
debug summaries, run receipts, and CLI presentation. Warnings are reserved for
|
||||||
|
process degradation or incomplete configured work. Data-quality findings are
|
||||||
|
advisories, and successful deterministic cleanup is recorded as observations.
|
||||||
|
An ordinary successful run therefore reports zero warnings while retaining
|
||||||
|
bounded diagnostic provenance for later review.
|
||||||
|
|
||||||
|
The framework aggregates findings deterministically by their stable identity
|
||||||
|
and complete pipeline origin, preserves exact occurrence counts, and retains
|
||||||
|
bounded representative samples. Warning groups fail rather than truncate;
|
||||||
|
advisory and observation representation is bounded with explicit truncation
|
||||||
|
metadata and exact unrepresented-occurrence counts.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
- `warnings.json` now uses the incompatible grouped
|
||||||
|
`notarius.warnings.v2` envelope and contains process warnings only. Consumers
|
||||||
|
of the former flat warning payload must migrate to the current
|
||||||
|
[JSON output contract](../integrations/json-output.md).
|
||||||
|
- The new `diagnostics.json` file uses `notarius.diagnostics.v1` and contains
|
||||||
|
advisory and observation groups. Production `index.json` files always expose
|
||||||
|
both `warnings_file` and `diagnostics_file`.
|
||||||
|
- The machine-readable run receipt is now `notarius.run-result.v2`. It replaces
|
||||||
|
`warning_count` with exact warning group and occurrence counts and adds
|
||||||
|
advisory/observation group, occurrence, and truncation fields. See the
|
||||||
|
current [run-result receipt](../integrations/run-result.md).
|
||||||
|
- Custom output modules must return their complete logical file set or an
|
||||||
|
error. The former `OutputResult.Warnings` field has been removed; an output
|
||||||
|
module cannot report a warning after serializing its output.
|
||||||
|
- Reusable state now uses `notarius.workspace.v4` and chunk-plan records use
|
||||||
|
`notarius.chunk-plan.v3` so they can preserve structured diagnostics. Older
|
||||||
|
pre-release reusable state is not reused under these contracts; start with
|
||||||
|
clean state when deterministic continuity with an older workspace is not
|
||||||
|
required.
|
||||||
|
- Validation acceptance, semantic retry budgets, rejection policy, and D&D
|
||||||
|
artifact schema identities are unchanged by this release.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
1. Update subprocess consumers to require `notarius.run-result.v2` and read
|
||||||
|
`warning_group_count`, `warning_occurrence_count`,
|
||||||
|
`diagnostic_group_count`, `diagnostic_occurrence_count`, and
|
||||||
|
`diagnostics_truncated`.
|
||||||
|
2. Update output-bundle consumers to decode `notarius.warnings.v2`, discover
|
||||||
|
`diagnostics.json` through `index.json`, and treat diagnostics as review
|
||||||
|
information rather than process warnings.
|
||||||
|
3. Update any custom output module for the removal of
|
||||||
|
`OutputResult.Warnings`; return an error when encoding cannot complete.
|
||||||
|
4. Clear pre-release reusable state before the first upgraded production run
|
||||||
|
when deterministic continuity with an older workspace is not required.
|
||||||
|
5. Run `notarius config validate --config <path> --pipeline <id>` and perform
|
||||||
|
one representative run before promoting the release in an automated
|
||||||
|
pipeline.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Added validated diagnostic dispositions, categories, origins, stable reason
|
||||||
|
codes, exact occurrence counts, and bounded representative samples.
|
||||||
|
- Added deterministic run-level aggregation with separate limits for
|
||||||
|
actionable warning groups and advisory/observation groups.
|
||||||
|
- Reclassified D&D source-relatedness and unresolved-identity findings as
|
||||||
|
data-quality advisories and routine normalization changes as observations.
|
||||||
|
- Preserved structured diagnostics across producer retries, validation,
|
||||||
|
generated-reference handoff, checkpoints, chunk-plan reuse, and debug
|
||||||
|
summaries while discarding superseded-attempt findings.
|
||||||
|
- Added grouped `warnings.json`, a new grouped `diagnostics.json`, and the
|
||||||
|
corresponding production index entries.
|
||||||
|
- Upgraded the machine-readable run receipt and human CLI summary to report
|
||||||
|
exact warning and diagnostic counts without allowing advisory volume to
|
||||||
|
create warning output.
|
||||||
|
- Removed post-encoding output warnings and hardened diagnostic validation,
|
||||||
|
overflow handling, aggregate memory bounds, and warning-file path
|
||||||
|
presentation.
|
||||||
|
- Documented diagnostic ownership, classification, operator interpretation,
|
||||||
|
durable contracts, and architectural invariants in ADR-0015 and the
|
||||||
|
canonical CLI, operations, integration, and internal documentation.
|
||||||
2474
docs/roadmap/archive/audit.md
Normal file
2474
docs/roadmap/archive/audit.md
Normal file
File diff suppressed because it is too large
Load Diff
648
docs/roadmap/archive/warning-signal-and-presentation-audit.md
Normal file
648
docs/roadmap/archive/warning-signal-and-presentation-audit.md
Normal file
@@ -0,0 +1,648 @@
|
|||||||
|
# Warning Signal And Presentation Audit
|
||||||
|
|
||||||
|
## Executive Assessment
|
||||||
|
|
||||||
|
Notarius warning execution is mechanically stronger than its operator-facing
|
||||||
|
presentation. Terminal-attempt promotion, stable ordering after concurrent
|
||||||
|
work, checkpoint replay, validation summaries, and debug retention are all
|
||||||
|
substantially correct. The audit found no general duplicate-append defect in
|
||||||
|
the extract, merge, or normalize handoffs and no leakage of abandoned-attempt
|
||||||
|
warnings into a successful result.
|
||||||
|
|
||||||
|
The warning channel itself is not coherent. One flat `contracts.Warning` type
|
||||||
|
currently represents at least four materially different concepts:
|
||||||
|
|
||||||
|
- actionable degradation or incomplete validation;
|
||||||
|
- heuristic data-quality doubt;
|
||||||
|
- successful but potentially reviewable fallback; and
|
||||||
|
- routine canonicalization, ordering, and deduplication observations.
|
||||||
|
|
||||||
|
That conflation is the primary reason successful runs produce a count that is
|
||||||
|
large but operationally weak. The maintained complete D&D example demonstrates
|
||||||
|
the problem without a live provider: an approved run with no rejected outputs
|
||||||
|
published 12 warning records, all from three advisory relatedness checks. An
|
||||||
|
operator separately reported a successful complete D&D run with 10 outputs,
|
||||||
|
one rejection, and 85 warnings. The production bundle for that run was not
|
||||||
|
available in this environment, so its reason-code distribution could not be
|
||||||
|
measured.
|
||||||
|
|
||||||
|
The current implementation also has four correctness or robustness gaps:
|
||||||
|
|
||||||
|
1. warning records lose stage, step, lane, module, validator, and chunk
|
||||||
|
provenance when promoted, which makes safe aggregation and diagnosis
|
||||||
|
impossible from `warnings.json` alone;
|
||||||
|
2. there is no framework-level validation or aggregate bound, and the NPC- and
|
||||||
|
spell-relatedness validators bypass the D&D warning limiter entirely;
|
||||||
|
3. warnings returned by an output encoder are added after `warnings.json` has
|
||||||
|
already been encoded, so the receipt, stderr, debug bundle, and published
|
||||||
|
warning file can disagree; and
|
||||||
|
4. a skipped validator contributes to `incomplete` validation but does not
|
||||||
|
receive the warning generated for an exhausted validator failure.
|
||||||
|
|
||||||
|
The recommended end state is a structured diagnostic contract with explicit
|
||||||
|
disposition, category, origin, occurrence count, and bounded samples. Warnings
|
||||||
|
are reserved for process-level degradation or incompleteness. LLM-judged or
|
||||||
|
deterministically inferred extraction-quality signals are advisories, never
|
||||||
|
warnings, and routine normalization observations remain inspectable without
|
||||||
|
being reported as top-level warnings. An ordinary successful run in which all
|
||||||
|
configured work completes normally should therefore report zero warnings. This
|
||||||
|
is an architectural and durable-contract change, not merely revised CLI prose.
|
||||||
|
|
||||||
|
## Evidence And Limits
|
||||||
|
|
||||||
|
The audit used:
|
||||||
|
|
||||||
|
- a complete static search of production `contracts.Warning` constructors,
|
||||||
|
reason-code constants, result fields, and promotion sites under `internal/`;
|
||||||
|
- call-path inspection through producer attempts, validators, lane
|
||||||
|
coordination, chunk-plan reuse, checkpoints, output encoding, debug output,
|
||||||
|
CLI presentation, and run-result construction;
|
||||||
|
- the maintained complete and minimal D&D examples with offline fake LLMs;
|
||||||
|
- focused deterministic tests for warning bounds, semantic-reconciliation
|
||||||
|
fallback, `warn_continue`, semantic retries, terminal rejection, concurrency
|
||||||
|
ordering, and checkpoint reuse; and
|
||||||
|
- the operator-provided observation of an 85-warning complete D&D run.
|
||||||
|
|
||||||
|
No provider-backed production run was attempted because this environment has
|
||||||
|
no API key. Consequently, the audit can establish warning mechanics, possible
|
||||||
|
multiplicity, synthetic volume, and obvious heuristic limitations, but cannot
|
||||||
|
estimate production frequency or the real false-positive rate of individual
|
||||||
|
D&D advisories. Those measurements are not required to choose the recommended
|
||||||
|
architecture; they are required before strengthening any heuristic advisory
|
||||||
|
into a rejection or setting a numerical production acceptance target.
|
||||||
|
|
||||||
|
## Complete Warning-Producer Inventory
|
||||||
|
|
||||||
|
### Framework And Generic Boundaries
|
||||||
|
|
||||||
|
| Producer | Reason code | Trigger and consequence | Multiplicity and bound | Current surfaces and coverage |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| Reference materialization in `internal/framework/pipeline/references.go` | `empty_reference` | A bound external reference is a valid, accepted media type but contains zero bytes. The prompt may receive materially incomplete context. | One per empty bound file; finite by configuration but no shared run-level cap. | Enters `RunInput.Warnings`; reference tests protect contextual scope. |
|
||||||
|
| Producer-attempt policy in `internal/framework/pipeline/producer_attempts.go` | `validator_execution_incomplete` | An applicable validator exhausted its execution budget and `warn_continue` accepted the otherwise valid candidate. | One per failed validator on each terminal candidate. An extract chain can multiply this by chunks and lanes. There is no global cap. | Durable warning, receipt count, stderr, debug, and validation summary. `TestWarnContinueRecordsOneWarningForEachExhaustedValidator` covers failures. |
|
||||||
|
| Chunk, extract, merge, normalize, and output module result contracts | Module-defined | A module may return arbitrary warnings with its successful candidate. | No contract validation, message limit, per-result cap, or global cap. Current production modules are inventoried below. | Terminal-attempt filtering and concurrency ordering are well tested. |
|
||||||
|
| Production JSON output encoder | None | The encoder copies incoming warnings into `warnings.json`; it does not currently create warnings. | Same count as its input. | JSON encoder and assembled-pipeline tests compare the incoming run warnings with the published file. |
|
||||||
|
| Output encoder result contract | Module-defined | Any output encoder may return warnings discovered during encoding. | Unbounded by contract. No production encoder currently exercises this capability. | Appended to final `RunOutput` only after logical files were encoded; this is the cross-surface defect described in AUD-WARN-004. |
|
||||||
|
|
||||||
|
Input adapters and production mergers do not currently have independent
|
||||||
|
warning producers. Chunk-plan and checkpoint decisions are structured manifest
|
||||||
|
or debug provenance rather than warnings. Cancellation and hard persistence,
|
||||||
|
reference, parsing, serialization, and provider failures remain errors.
|
||||||
|
|
||||||
|
### D&D Extraction Gates
|
||||||
|
|
||||||
|
| Producer | Reason code | Trigger and consequence | Multiplicity and bound |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `dnd/combat-turns` extractor | `scene_classification_unavailable` | The chunk has no exact matching scene-description classification. The extractor returns an empty accepted result and skips the LLM, so combat-turn output may be incomplete. | At most one per chunk for this lane. |
|
||||||
|
| `dnd/enemy-events` extractor | `scene_classification_unavailable` | The same missing or mismatched scene gate causes accepted empty enemy-event output. | At most one per chunk for this lane. |
|
||||||
|
|
||||||
|
An exact non-combat classification produces an intentional empty result without
|
||||||
|
a warning. An exact combat classification proceeds normally. The two producers
|
||||||
|
share a code and operator consequence but use different messages; their module
|
||||||
|
origins are not retained in the final warning record.
|
||||||
|
|
||||||
|
### D&D Source-Relatedness Validators
|
||||||
|
|
||||||
|
All ten relatedness validators are deterministic advisories: they approve the
|
||||||
|
candidate and warn when contextual prose or an entity name is not lexically
|
||||||
|
present in cited text. Shape and source-reference failures are deliberately
|
||||||
|
left to blocking validators earlier in the chain. The same relatedness
|
||||||
|
validator is registered in both the extract and normalize default chain for
|
||||||
|
each artifact family in `internal/modules/dnd/register/chains.go`.
|
||||||
|
|
||||||
|
| Artifact family | Warning reason | Per-record trigger | Local bound | Omission reason |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| Combat turns | `combat_turn_not_near_source` | Actor token sequence absent | 20 per validator invocation | `combat_turn_relatedness_warnings_omitted` |
|
||||||
|
| Enemy events | `enemy_event_not_near_source` | Subject token sequence absent | 20 | `enemy_event_relatedness_warnings_omitted` |
|
||||||
|
| Item occurrences | `item_occurrence_source_unrelated` | Item name token sequence absent | 20 | `item_occurrence_relatedness_warnings_omitted` |
|
||||||
|
| Item registry | `item_not_near_source` | Item name token sequence absent | 20 | `item_relatedness_warnings_omitted` |
|
||||||
|
| Location occurrences | `location_occurrence_not_near_source` | Location name token sequence absent | 20 | `location_occurrence_relatedness_warnings_omitted` |
|
||||||
|
| Location registry | `location_not_near_source` | Location name token sequence absent | 20 | `location_relatedness_warnings_omitted` |
|
||||||
|
| NPC occurrences | `npc_occurrence_not_near_source` | NPC name token sequence absent | 20 | `npc_occurrence_relatedness_warnings_omitted` |
|
||||||
|
| NPC registry | `npc_not_near_source` | NPC name token sequence absent | **Unbounded** | None |
|
||||||
|
| Scene descriptions | `scene_description_not_near_source` | No significant title or summary token appears; up to two findings per scene | 20 | `scene_description_relatedness_warnings_omitted` |
|
||||||
|
| Spells | `spell_not_near_source` | Spell-name token sequence absent | **Unbounded** | None |
|
||||||
|
|
||||||
|
The eight limiter-generated omission records are presentation artifacts, not
|
||||||
|
new source-relatedness conditions. They occupy a warning slot and make list
|
||||||
|
length differ from the actual occurrence count.
|
||||||
|
|
||||||
|
### D&D Normalizers
|
||||||
|
|
||||||
|
Every production D&D normalizer bounds its returned warning slice to 20 through
|
||||||
|
`internal/modules/dnd/shared/diagnostics`, including a final omission record
|
||||||
|
when needed. Registry semantic retries reserve space for their fallback
|
||||||
|
warning. The following table is complete by semantically distinct condition;
|
||||||
|
codes listed together are parallel artifact-family variants.
|
||||||
|
|
||||||
|
| Condition | Reason codes | Result impact | Current classification assessment |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Display or field whitespace/name canonicalization | `npc_fields_normalized`, `item_fields_normalized`, `location_fields_normalized`, `spell_name_canonicalized`, `combat_actor_canonicalized`, `enemy_event_name_canonicalized`, `item_occurrence_name_canonicalized`, `location_occurrence_name_canonicalized`, `scene_description_prose_normalized` | Deterministic successful mutation. The item-occurrence code can also describe `from`/`to` whitespace, not only the item name. | Routine observation. |
|
||||||
|
| Durable ID recomputation | `npc_id_recomputed`, `item_id_recomputed`, `location_id_recomputed` | Restores the deterministic name-derived ID. | Routine observation; invalid identity is separately rejected by default chains. |
|
||||||
|
| Source-reference sorting or deduplication | `source_references_normalized` | Sorts and removes exact duplicate references while deliberately preserving invalid references for their validators. | Routine observation. Shared code is useful but ambiguous without producer origin. |
|
||||||
|
| Canonical record ordering | `combat_turns_reordered`, `enemy_events_reordered`, `item_occurrences_reordered`, `location_occurrences_reordered`, `npc_occurrences_reordered`, `scene_description_order_normalized` | Deterministic order changes only. | Routine observation. |
|
||||||
|
| Exact or approved semantic duplicate consolidation | `duplicate_npc_collapsed`, `duplicate_item_collapsed`, `duplicate_location_collapsed`, `duplicate_spell_cast_collapsed`, `duplicate_combat_turn_collapsed`, `duplicate_enemy_event_collapsed`, `duplicate_item_occurrence_collapsed`, `duplicate_location_occurrence_collapsed`, `duplicate_npc_occurrence_collapsed`, `scene_description_duplicate_collapsed` | Removes duplicate records and preserves or combines canonical evidence according to the artifact policy. Registry codes cover both exact and accepted semantic consolidation. | Durable normalization observation; not normally operator-actionable. |
|
||||||
|
| Unresolved external membership | `spell_name_unresolved`, `item_occurrence_unknown_item_id`, `location_occurrence_unknown_location_id` | The value is preserved but is not grounded in the effective catalog or registry. Default chains normally reject the same condition before normalization; it remains reachable with validator overrides or defensive direct use. | Actionable data-quality warning. |
|
||||||
|
| Unsafe currency consolidation proposal | `item_semantic_proposal_invalid` | The proposed group is rejected and all records are preserved because denominations or currency/non-currency members are incompatible. The same code is also used internally as a retry reason. | Advisory about model proposal quality; no accepted-data loss. The control and diagnostic meanings should be separated. |
|
||||||
|
| Semantic reconciliation unavailable or exhausted | `npc_semantic_reconciliation_exhausted`, `item_semantic_reconciliation_exhausted`, `location_semantic_reconciliation_exhausted` | The safe deterministic result is accepted, but possible semantic duplicates remain. | Actionable fallback warning. |
|
||||||
|
| Local warning truncation | `npc_normalization_warnings_omitted`, `item_normalization_warnings_omitted`, `location_normalization_warnings_omitted`, `spell_normalization_warnings_omitted`, `combat_turn_normalization_warnings_omitted`, `enemy_event_normalization_warnings_omitted`, `item_occurrence_normalization_warnings_omitted`, `location_occurrence_normalization_warnings_omitted`, `npc_occurrence_normalization_warnings_omitted`, `scene_description_normalization_warnings_omitted` | Reports that individual records were omitted from presentation. | Group metadata, not an independent warning. |
|
||||||
|
|
||||||
|
No production D&D normalization warning exposes raw model responses,
|
||||||
|
correction guidance, or provider errors. Most dynamic names are quoted and
|
||||||
|
truncated by the shared helper. That local discipline is not enforced by the
|
||||||
|
generic warning contract, and the spell relatedness message does not use the
|
||||||
|
shared truncation helper.
|
||||||
|
|
||||||
|
## Warning Propagation And Surface Map
|
||||||
|
|
||||||
|
```text
|
||||||
|
external-reference warnings -----------------------------+
|
||||||
|
|
|
||||||
|
module candidate warnings -> validation chain warnings |
|
||||||
|
| | |
|
||||||
|
+---- producer-attempt terminal policy -------+
|
||||||
|
| |
|
||||||
|
accepted / terminal rejection only |
|
||||||
|
| |
|
||||||
|
chunk or lane result in canonical order |
|
||||||
|
| |
|
||||||
|
checkpoint record/replay and ordered step merge |
|
||||||
|
| |
|
||||||
|
RunOutput.Warnings <------------+
|
||||||
|
|
|
||||||
|
OutputRequest -> output encoder
|
||||||
|
| |
|
||||||
|
warnings.json OutputResult.Warnings
|
||||||
|
|
|
||||||
|
appended to final RunOutput only
|
||||||
|
|
|
||||||
|
receipt, stderr, final debug warning summary
|
||||||
|
```
|
||||||
|
|
||||||
|
### Attempts And Validation
|
||||||
|
|
||||||
|
- `runProducerAttempts` promotes only the terminal accepted or terminal
|
||||||
|
rejected candidate's module and completed-validator warnings. Operational,
|
||||||
|
structural, semantic, and module-directed attempts that are superseded are
|
||||||
|
retained in attempt debug artifacts but not in the final collection.
|
||||||
|
- A module-directed semantic-reconciliation retry adds its fallback warning
|
||||||
|
only when no retry remains. Earlier attempt warnings are discarded.
|
||||||
|
- On `warn_continue`, warnings from the otherwise accepted candidate and
|
||||||
|
completed approved or rejected validators are retained. One fixed,
|
||||||
|
non-sensitive `validator_execution_incomplete` warning is added for every
|
||||||
|
failed validator. Skipped validators affect the validation summary and final
|
||||||
|
`incomplete` status but do not receive such a warning.
|
||||||
|
- A semantic terminal rejection retains only warnings from that rejected
|
||||||
|
attempt. Structural rejection after producer failure cannot retain a
|
||||||
|
candidate warning because no valid candidate result exists.
|
||||||
|
|
||||||
|
These behaviors are protected by the producer-attempt, extract-handoff,
|
||||||
|
rejection-warning, normalize-retry, and attempt-debug tests. They are the right
|
||||||
|
foundation for the redesign and should not be replaced with early-exit or
|
||||||
|
all-attempt accumulation.
|
||||||
|
|
||||||
|
### Concurrency And Ordering
|
||||||
|
|
||||||
|
Extract jobs are dispatched chunk-first and lane-second. Results are stored by
|
||||||
|
chunk index, finalized in ascending chunk order, and lane continuations are
|
||||||
|
merged into a slice indexed by configured lane order. Pipeline steps run in
|
||||||
|
configured order. The resulting public order is therefore:
|
||||||
|
|
||||||
|
1. pre-run reference warnings;
|
||||||
|
2. chunk-stage warnings;
|
||||||
|
3. step order;
|
||||||
|
4. configured lane order within each step;
|
||||||
|
5. chunk order within extract;
|
||||||
|
6. merge warnings; then
|
||||||
|
7. normalize warnings; followed by any output-result warnings.
|
||||||
|
|
||||||
|
`TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion` exercises warning
|
||||||
|
order under reversed completion. No completion-order leak was found.
|
||||||
|
|
||||||
|
### Chunk Plans And Checkpoints
|
||||||
|
|
||||||
|
- A reusable chunk plan stores producer warnings only. Current validators run
|
||||||
|
again, and their current warnings are appended. Warnings from a cached plan
|
||||||
|
candidate that fails current validation are discarded before regeneration.
|
||||||
|
- Accepted extract, merge, and normalize checkpoints store the terminal
|
||||||
|
warnings for their stage. Reuse loads and appends those warnings once at the
|
||||||
|
same logical handoff. Tests compare fresh and resumed warning collections and
|
||||||
|
preserve their order.
|
||||||
|
- Validation-incomplete accepted outputs are not reusable, preventing a later
|
||||||
|
run from silently treating incomplete validation as complete.
|
||||||
|
- Required accepted-normalize hydration replays that normalize checkpoint's
|
||||||
|
warnings; checkpoint decisions separately expose that reuse occurred.
|
||||||
|
|
||||||
|
The recommended redesign should keep fresh and resumed logical diagnostics
|
||||||
|
equivalent. Whether a result was reused belongs in checkpoint provenance, not
|
||||||
|
in the diagnostic grouping key; adding a `reused` distinction would fragment
|
||||||
|
groups and make equivalent runs present differently.
|
||||||
|
|
||||||
|
### Terminal Surfaces
|
||||||
|
|
||||||
|
| Surface | Current content | Audience | Audit result |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `RunOutput.Warnings` | Flat final slice | Framework and CLI | Canonical in-memory list, but lacks origin and bounds. |
|
||||||
|
| Published `warnings.json` | Object containing the warnings passed into the output encoder | Durable consumers | Exact for the production JSON encoder unless the encoder itself returns warnings. |
|
||||||
|
| `index.json` | Path to `warnings.json` | Durable consumers | Stable discovery path; no separate diagnostic-detail path. |
|
||||||
|
| Run-result v1 | `warning_count = len(final RunOutput.Warnings)` | Subprocess callers | Count only; no group/occurrence distinction. |
|
||||||
|
| Human stderr | `run completed with N warning(s)` | Operators | Count only and no direct detail path. Successful exit remains zero. |
|
||||||
|
| Manifest | Validation and rejection summaries, no warning collection | Durable provenance | Correctly avoids duplicating the flat list. |
|
||||||
|
| Debug summary `warnings.json` | Raw final warning array | Operators/developers | Includes final output-result warnings and can therefore differ from published `warnings.json`. |
|
||||||
|
| Debug run report | Final warning count | Operators/developers | Same final slice length as receipt and stderr. |
|
||||||
|
| Attempt/stage debug | Candidate-local warning detail and origin in path/envelope | Forensics | Sufficient to diagnose provenance, but debug capture is optional and is not a durable consumer contract. |
|
||||||
|
|
||||||
|
## Empirical Measurements
|
||||||
|
|
||||||
|
### Offline And Synthetic Runs
|
||||||
|
|
||||||
|
| Scenario | Result | What it establishes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Maintained complete D&D config and transcript with the repository's offline fake LLM | Approved, 10 normalized outputs, 0 rejected outputs, 12 warnings; receipt, stderr, and published file all reported 12 | An ordinary structurally successful workflow can be noisy without fallback or incomplete validation. |
|
||||||
|
| Same complete run, grouped after publication | Three reason codes, seven exact `(reason, scope, message)` tuples, maximum exact-tuple repetition of three | The flat count materially overstates distinct operator conditions. Scope resets within chunks and does not identify origin. |
|
||||||
|
| Maintained focused scene-description workflow | Approved, one normalized output, 0 warnings | The warning channel can be quiet when synthetic model text is lexically grounded. |
|
||||||
|
| Generic warning publication contract | One warning reaches successful stderr, durable output, and debug summary | The ordinary pre-output path is consistent. |
|
||||||
|
| NPC semantic-reconciliation candidate-limit fallback | No LLM call, all records preserved, one exhaustion warning, total warnings no greater than 20 | Fallback is bounded and materially different from routine normalization. |
|
||||||
|
| `warn_continue` with two failed validators and one skipped validator | Validation status contains all three incomplete validators; warning slice contains two execution-incomplete records | Current warning count does not describe all incomplete validation. |
|
||||||
|
| Retrying extract candidate | Two producer attempts; only the accepted attempt's one warning is final | Retry does not amplify abandoned warnings. |
|
||||||
|
| Terminal semantic rejection | Only the final rejected attempt's operation and validator warnings are final | Rejection diagnostics are retained without retaining superseded warnings. |
|
||||||
|
| Fresh versus reused extract checkpoint | Warning collections are deeply equal | Checkpoint replay does not itself amplify warnings. |
|
||||||
|
| Spell normalizer with 21 unresolved entries | 20 records: 19 samples plus one omission record saying two additional warnings were omitted | `warning_count` is neither exact occurrence count nor distinct-condition count. |
|
||||||
|
|
||||||
|
The focused audit tests passed in `internal/cli`,
|
||||||
|
`internal/framework/pipeline`, the NPC-registry and spell normalizers, and all
|
||||||
|
D&D packages.
|
||||||
|
|
||||||
|
### Bounded Sample Review
|
||||||
|
|
||||||
|
The complete offline D&D run produced:
|
||||||
|
|
||||||
|
| Reason | Count | Sample | Review |
|
||||||
|
| --- | ---: | --- | --- |
|
||||||
|
| `location_not_near_source` | 4 | `Moon Gate` was absent from cited text | Correctly identifies deliberately unsupported fake output. Three records shared the same exact tuple because chunk and stage origin were lost. |
|
||||||
|
| `location_occurrence_not_near_source` | 4 | A `Moon Gate` visit was absent from cited text | Correctly identifies the same unsupported registry-driven occurrence, but repeats the same operator concern across extraction and normalization. |
|
||||||
|
| `scene_description_not_near_source` | 4 | A title or summary had no significant exact token in cited text | Mixed value. Generic `session scene` prose is ungrounded, while `Arrival` versus transcript `arrive` illustrates an expected lexical false positive. |
|
||||||
|
|
||||||
|
This fake workflow is an integration fixture, not a model-quality benchmark.
|
||||||
|
It nonetheless proves that the checks carry useful evidence while being too
|
||||||
|
imprecise and repetitive to serve as one-warning-per-record operator alerts.
|
||||||
|
|
||||||
|
### Production Evidence Still Needed
|
||||||
|
|
||||||
|
The reported 85-warning run establishes that high volume occurs in practice,
|
||||||
|
but the following remain unknown:
|
||||||
|
|
||||||
|
- dominant production reason codes and stage/lane sources;
|
||||||
|
- unique group count versus repeated occurrence count;
|
||||||
|
- false-positive rate for each relatedness family;
|
||||||
|
- how much volume comes from normalization observations versus advisories;
|
||||||
|
- whether fresh and resumed production runs remain equivalent; and
|
||||||
|
- a defensible numerical acceptance target.
|
||||||
|
|
||||||
|
If further data is worthwhile, the operator can supply the v1 receipt,
|
||||||
|
`warnings.json`, and manifest validation summaries without supplying transcript
|
||||||
|
or lane artifacts. An initial privacy-preserving report should group by reason
|
||||||
|
code and normalized scope family, count exact repeated tuples, and omit message
|
||||||
|
text. Reviewing heuristic precision requires a separately approved bounded
|
||||||
|
sample with its cited source context.
|
||||||
|
|
||||||
|
## Classification Of Current Conditions
|
||||||
|
|
||||||
|
| Target disposition | Current families | Result impact | Operator action | Durable placement |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| **Warning** | Empty reference; validator failure or skip accepted under `warn_continue`; unavailable required scene classification; exhausted semantic reconciliation | A configured process completed under an allowed degraded or incomplete policy rather than completing normally | Correct reference/configuration, inspect provider/validator, or rerun | Actionable `warnings.json`, receipt summary, stderr summary, debug |
|
||||||
|
| **Advisory** | Source-relatedness heuristics; unresolved spell or registry membership; guarded invalid semantic proposal; any future LLM-judged uncertainty or extraction-quality signal | Uncertain data quality or poor model proposal, but accepted data is structurally valid and deterministic guards prevented unsafe mutation | Optional model/source review; no routine action for every record | Durable diagnostic detail and debug; never a top-level warning |
|
||||||
|
| **Observation** | Whitespace/name/ID/source-reference canonicalization; canonical ordering; exact and approved semantic duplicate consolidation | Successful intended normalization | None under normal operation | Durable bounded normalization diagnostics or debug; no stderr warning |
|
||||||
|
| **Not a diagnostic** | Rejection, invalid structure, cancellation, persistence error, provider failure under fail-run policy | Candidate or run did not complete according to policy | Inspect rejection/error and retry or correct input/configuration | Existing rejection, validation summary, error, and debug contracts |
|
||||||
|
|
||||||
|
Exact and semantic duplicate consolidation should remain distinguishable in
|
||||||
|
category or reason metadata even though both are observations. Semantic
|
||||||
|
reconciliation exhaustion remains a warning because a capability was not
|
||||||
|
applied; successful approved consolidation is an observation because it is the
|
||||||
|
normalizer's intended work.
|
||||||
|
|
||||||
|
## Ranked Findings
|
||||||
|
|
||||||
|
### AUD-WARN-001 — The Flat Warning Type Destroys Signal Quality
|
||||||
|
|
||||||
|
- **Priority:** High operator impact; high implementation leverage.
|
||||||
|
- **Evidence:** `contracts.Warning` has only scope, reason, and message. Routine
|
||||||
|
normalizer changes, heuristic doubt, fallback, and incomplete validation all
|
||||||
|
enter the same slice and the same CLI count. The offline complete run's 12
|
||||||
|
records were all advisories; the operator observed 85 records in a successful
|
||||||
|
real run.
|
||||||
|
- **Impact:** Operators cannot tell whether a warning requires a rerun, a
|
||||||
|
configuration repair, optional review, or no action. Repeated routine output
|
||||||
|
trains them to ignore the channel.
|
||||||
|
- **Recommendation:** Replace the flat semantic contract with explicit
|
||||||
|
`warning`, `advisory`, and `observation` dispositions plus a small category
|
||||||
|
vocabulary. Do not infer disposition from message text or require every
|
||||||
|
downstream consumer to maintain a reason-code policy table.
|
||||||
|
|
||||||
|
### AUD-WARN-002 — Warning Records Lose The Origin Needed For Diagnosis And Aggregation
|
||||||
|
|
||||||
|
- **Priority:** High correctness and usability impact.
|
||||||
|
- **Evidence:** The runner knows stage, step, lane, module, validator, chunk ID,
|
||||||
|
and chunk index at promotion time, but `terminalWarnings` flattens module and
|
||||||
|
validator records into `[]contracts.Warning`. Per-chunk scopes such as
|
||||||
|
`locations[0]` and `occurrences[0]` then repeat without identifying their
|
||||||
|
chunk or producer. `source_references_normalized` is intentionally shared
|
||||||
|
across families and is therefore especially ambiguous.
|
||||||
|
- **Impact:** `warnings.json` cannot answer which stage or module produced a
|
||||||
|
record. Message- or scope-based deduplication would merge unrelated findings
|
||||||
|
or retain accidental duplicates.
|
||||||
|
- **Recommendation:** Keep module findings free of framework context, then have
|
||||||
|
the framework add a structured origin envelope before promotion. Validator
|
||||||
|
findings must retain validator identity instead of passing through
|
||||||
|
`validationReport.Warnings()` as a flat slice.
|
||||||
|
|
||||||
|
### AUD-WARN-003 — Warning Volume Is Not End-To-End Bounded Or Validated
|
||||||
|
|
||||||
|
- **Priority:** High robustness impact; medium immediate likelihood.
|
||||||
|
- **Evidence:** D&D's `LimitWarnings` caps most individual producers at 20, but
|
||||||
|
NPC- and spell-relatedness return one warning per record without the helper.
|
||||||
|
Every extract validator is invoked per chunk, all ten relatedness checks run
|
||||||
|
again after normalization, and there is no run-level collector. The generic
|
||||||
|
contract validates neither disposition nor reason/message size, UTF-8,
|
||||||
|
blankness, or total records.
|
||||||
|
- **Impact:** Warning memory and output grow with chunks, lanes, configured
|
||||||
|
validators, and record counts. Local omission records lose exact occurrence
|
||||||
|
semantics while still incrementing `warning_count`.
|
||||||
|
- **Recommendation:** Add a generic bounded diagnostic collector that preserves
|
||||||
|
exact occurrence counts and bounded samples. Validate all diagnostic fields
|
||||||
|
at the module/framework boundary. Immediately bring NPC and spell
|
||||||
|
relatedness under the existing cap if the full redesign is staged.
|
||||||
|
|
||||||
|
### AUD-WARN-004 — Output Encoder Warnings Make Durable Surfaces Disagree
|
||||||
|
|
||||||
|
- **Priority:** Medium current impact; high contract correctness risk.
|
||||||
|
- **Evidence:** `Runner.Run` passes existing warnings to `encoder.Encode`, then
|
||||||
|
the production encoder serializes `warnings.json`. Only after encoding does
|
||||||
|
the runner append `OutputResult.Warnings`. The receipt, stderr, debug summary,
|
||||||
|
and debug run report see the final slice; the already-created published file
|
||||||
|
cannot. No production encoder currently returns a warning, so ordinary JSON
|
||||||
|
runs do not trigger the defect.
|
||||||
|
- **Impact:** A valid output-module implementation can violate the documented
|
||||||
|
claim that `warning_count` describes the published warning collection.
|
||||||
|
- **Recommendation:** Remove successful output warnings from the output-module
|
||||||
|
contract unless a demonstrated use case requires them; encoding failures
|
||||||
|
should be errors and optional encoder observations should be debug data. A
|
||||||
|
two-phase finalize API is the viable but more complex alternative.
|
||||||
|
|
||||||
|
### AUD-WARN-005 — Validation Skips Are Incomplete But Not Warned
|
||||||
|
|
||||||
|
- **Priority:** Medium operator/correctness impact.
|
||||||
|
- **Evidence:** `firstIncompleteValidation` treats failed and skipped validators
|
||||||
|
alike, and validation summaries include both. `incompleteValidationWarnings`
|
||||||
|
emits records only for `validationFailed`. The focused test demonstrates
|
||||||
|
three incomplete validators but two warnings.
|
||||||
|
- **Impact:** A successful run can have `validation_status: incomplete` while
|
||||||
|
its warning count understates or even omits the affected validators. A caller
|
||||||
|
that checks only warnings receives a weaker signal than the manifest and
|
||||||
|
receipt status.
|
||||||
|
- **Recommendation:** Produce one aggregated incomplete-validation warning
|
||||||
|
group whose occurrences cover both failure and skip, while retaining typed
|
||||||
|
outcome and safe reason metadata in the validation summary. Do not expose
|
||||||
|
provider errors or arbitrary skip prose in model or operator messages.
|
||||||
|
|
||||||
|
### AUD-WARN-006 — Relatedness Checks Are Useful But Repetitive And Lexically Weak
|
||||||
|
|
||||||
|
- **Priority:** Medium operator impact; low acceptance-policy urgency.
|
||||||
|
- **Evidence:** Every family runs the advisory in both extract and normalize
|
||||||
|
chains. The complete fixture contains exact repeated tuples, and the checks
|
||||||
|
rely on exact normalized token sequences or a minimal significant-token
|
||||||
|
overlap. Reason naming drifts between `*_not_near_source` and
|
||||||
|
`*_source_unrelated`.
|
||||||
|
- **Impact:** The checks can catch unsupported entities, but aliases, pronouns,
|
||||||
|
inflection, and generic scene prose create predictable false positives. Flat
|
||||||
|
per-record presentation magnifies them.
|
||||||
|
- **Recommendation:** Retain the validators and their stage-local execution,
|
||||||
|
but classify and aggregate them as advisories. Normalize reason-code naming
|
||||||
|
when the diagnostic contract changes. Do not strengthen them into rejection
|
||||||
|
rules without a human-reviewed production evaluation.
|
||||||
|
|
||||||
|
### AUD-WARN-007 — `warning_count` Has No Stable Operational Meaning
|
||||||
|
|
||||||
|
- **Priority:** High downstream-contract impact.
|
||||||
|
- **Evidence:** The receipt and CLI use `len(output.Warnings)`. One list element
|
||||||
|
can be an omission summary representing several hidden occurrences; repeated
|
||||||
|
records can represent the same condition; and skipped validators can be
|
||||||
|
absent. A 21-occurrence spell test produces a list length of 20.
|
||||||
|
- **Impact:** The value is neither an exact occurrence count nor a distinct
|
||||||
|
warning-group count. Consumers cannot set policy or present a trustworthy
|
||||||
|
summary from it.
|
||||||
|
- **Recommendation:** Introduce explicit warning-group and warning-occurrence
|
||||||
|
counts in a versioned receipt. Do not silently redefine the v1 field.
|
||||||
|
|
||||||
|
## Recommended Target Contract And Presentation Model
|
||||||
|
|
||||||
|
### Diagnostic Model
|
||||||
|
|
||||||
|
Use one validated internal diagnostic model with these concepts:
|
||||||
|
|
||||||
|
- **disposition:** `warning`, `advisory`, or `observation`;
|
||||||
|
- **category:** a small enum such as `configuration`, `degradation`,
|
||||||
|
`validation_incomplete`, `data_quality`, `fallback`, or `normalization`;
|
||||||
|
- **reason code:** stable semantic identity owned by the producer;
|
||||||
|
- **origin:** framework-added phase/stage, step ID, lane ID, module key,
|
||||||
|
validator name, chunk ID, and chunk index when applicable;
|
||||||
|
- **occurrence count:** exact number of matching findings;
|
||||||
|
- **samples:** a small deterministic list of bounded scope/message pairs; and
|
||||||
|
- **omitted sample count:** `occurrence_count - len(samples)`, represented as
|
||||||
|
metadata rather than another diagnostic record.
|
||||||
|
|
||||||
|
Errors and rejected outputs must not become diagnostic dispositions. A warning
|
||||||
|
means that the run completed under policy despite a process-level degradation
|
||||||
|
or incomplete configured operation. Advisory and observation dispositions can
|
||||||
|
describe accepted artifact quality and transformation provenance, but no
|
||||||
|
LLM-judged extraction-quality signal may be promoted to a warning. Validation
|
||||||
|
status remains authoritative for approval, rejection, and incomplete
|
||||||
|
validation.
|
||||||
|
|
||||||
|
### Aggregation
|
||||||
|
|
||||||
|
The framework runner should own aggregation after it enriches findings with
|
||||||
|
origin and before public output construction. Modules and validators retain
|
||||||
|
semantic ownership of disposition, category, reason, scope, and message; they
|
||||||
|
must not own CLI or file presentation.
|
||||||
|
|
||||||
|
The default stable key should be:
|
||||||
|
|
||||||
|
```text
|
||||||
|
disposition + category + reason_code
|
||||||
|
+ phase/stage + step_id + lane_id + module_key + validator_name
|
||||||
|
```
|
||||||
|
|
||||||
|
Chunk, record scope, and message text belong in samples and must not be part of
|
||||||
|
the group key. This groups repeated per-chunk findings without merging the same
|
||||||
|
code across distinct producers or pipeline locations. Group order should be
|
||||||
|
the first occurrence in the runner's existing canonical order; sample order
|
||||||
|
should follow the same order. A final canonical sort by the complete origin key
|
||||||
|
is also viable, but completion timing must never choose either order.
|
||||||
|
|
||||||
|
Aggregation must be incremental and bounded. Producers should use a shared
|
||||||
|
collector that counts every occurrence while retaining only bounded samples;
|
||||||
|
the framework then merges producer groups without reconstructing counts from
|
||||||
|
omission prose. A global maximum group count is also required, with overflow
|
||||||
|
represented by structured aggregate metadata and with actionable groups given
|
||||||
|
priority over lower dispositions.
|
||||||
|
|
||||||
|
### Durable Files
|
||||||
|
|
||||||
|
Keep one canonical home for each class:
|
||||||
|
|
||||||
|
- `warnings.json` should contain versioned, grouped actionable warnings only;
|
||||||
|
- a new `diagnostics.json` should contain versioned advisory and observation
|
||||||
|
groups only, avoiding duplication of warning groups;
|
||||||
|
- `index.json` should link both files;
|
||||||
|
- `rejected.json` and manifest validation summaries should retain their current
|
||||||
|
separate responsibilities; and
|
||||||
|
- debug bundles should retain candidate-attempt detail plus the final grouped
|
||||||
|
projections.
|
||||||
|
|
||||||
|
This is preferable to keeping all detail in `warnings.json` and filtering only
|
||||||
|
the CLI: downstream consumers would otherwise continue to receive a semantically
|
||||||
|
mixed warning contract, and routine observations would still dominate the
|
||||||
|
durable file.
|
||||||
|
|
||||||
|
### CLI And Receipt
|
||||||
|
|
||||||
|
For a successful human run with actionable warnings, print a concise summary
|
||||||
|
such as:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius: run completed with 2 warning groups (7 occurrences); details=/.../warnings.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Advisories and observations should not produce the warning line. Their durable
|
||||||
|
path remains discoverable through `index.json`; a concise non-warning count can
|
||||||
|
be added to the ordinary success line only if operator testing shows value. An
|
||||||
|
ordinary successful run with no process degradation should write nothing to
|
||||||
|
the warning stream even when it publishes quality advisories or normalization
|
||||||
|
observations.
|
||||||
|
|
||||||
|
Create `notarius.run-result.v2` rather than redefining v1. It should expose at
|
||||||
|
least:
|
||||||
|
|
||||||
|
- `warning_group_count`;
|
||||||
|
- `warning_occurrence_count`; and
|
||||||
|
- `diagnostic_group_count` for non-warning durable groups.
|
||||||
|
|
||||||
|
The receipt should continue to expose validation status, validation summaries,
|
||||||
|
and rejected-output count independently. Process exit behavior should not
|
||||||
|
change as part of warning presentation reform.
|
||||||
|
|
||||||
|
### Checkpoint Semantics
|
||||||
|
|
||||||
|
Store the structured terminal diagnostic groups with accepted checkpoints and
|
||||||
|
replay them exactly once at their logical stage. Fresh and reused runs should
|
||||||
|
produce the same public groups and counts. Checkpoint events and debug records,
|
||||||
|
not diagnostic identity, should disclose whether computation was reused.
|
||||||
|
|
||||||
|
### Output Encoder Boundary
|
||||||
|
|
||||||
|
Prefer removing `OutputResult.Warnings`. A successful output encoder should
|
||||||
|
either return the complete logical files or fail. If future encoders genuinely
|
||||||
|
need to produce durable post-encoding warnings, introduce an explicit
|
||||||
|
two-phase prepare/finalize contract so those warnings can be included in the
|
||||||
|
same published collection. Do not retain the current self-inconsistent
|
||||||
|
one-pass capability.
|
||||||
|
|
||||||
|
## Resolution Of Required Design Questions
|
||||||
|
|
||||||
|
| Question | Recommendation | Viable alternative and tradeoff |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Explicit severity/disposition or external reason mapping? | Put validated disposition and category in the contract. | A central reason-code registry avoids payload fields but makes new modules depend on a second synchronized policy table and leaves downstream meaning implicit. |
|
||||||
|
| Keep all detail in `warnings.json` or separate it? | Separate grouped actionable warnings from grouped advisories/observations in `diagnostics.json`. | Keep the flat durable list and aggregate only CLI output; simpler migration, but it preserves the noisy downstream contract and ambiguous count. |
|
||||||
|
| Who owns aggregation? | Framework runner/coordinator after origin enrichment. | Output module aggregation keeps framework types smaller but duplicates policy across encoders and cannot repair missing validator origin. |
|
||||||
|
| Stable aggregation key? | Disposition, category, reason code, and full producer origin; exclude chunk/scope/message. | Explicit producer-supplied grouping keys offer flexibility but add another identity that can drift from reason codes. Message-template grouping is brittle and unsafe. |
|
||||||
|
| Samples and omissions? | Exact occurrence count plus deterministic bounded samples and numeric omitted-sample count. | Omission warning records preserve the current representation but inflate group counts and require prose parsing. |
|
||||||
|
| `warning_count` semantics? | Version receipt and replace ambiguity with group and occurrence counts. | Keep v1 count as published record length and add optional fields; compatible, but two competing warning counts remain easy to misuse. |
|
||||||
|
| Which normalization changes remain warnings? | Only exhausted process fallback. Unresolved membership is a data-quality advisory; successful canonicalization, reordering, ID repair, source-ref dedupe, and duplicate consolidation are observations. | Treat unresolved membership or semantic duplicate consolidation as warnings because they affect grounding or cardinality; more conservative, but it violates the process-only warning rule and reports accepted artifact quality as an operational failure. |
|
||||||
|
| Source-relatedness disposition? | Grouped advisory by default; preserve current approve behavior. | Retain warning disposition or make rejection configurable. Rejection requires production precision evidence; current lexical rules are not strong enough. |
|
||||||
|
| Checkpoint-loaded warnings? | Present the same logical groups as fresh execution and use checkpoint events for reuse provenance. | Mark groups as replayed; aids forensics but fragments aggregation and makes semantically equivalent runs differ. |
|
||||||
|
| ADR and schema versions? | Add an ADR and version the run receipt and diagnostic files. | Treat the work as CLI-only presentation and avoid an ADR; insufficient because module contracts, output files, checkpoint payloads, and downstream fields change. |
|
||||||
|
|
||||||
|
## Compatibility, Documentation, And ADR Implications
|
||||||
|
|
||||||
|
The target alters public and internal contracts enough to require a new ADR.
|
||||||
|
It should record:
|
||||||
|
|
||||||
|
- the distinction among warnings, advisories, observations, rejections, and
|
||||||
|
errors;
|
||||||
|
- the invariant that warnings are process-level signals, LLM-judged extraction
|
||||||
|
quality is never a warning, and ordinary non-degraded success has zero
|
||||||
|
warnings;
|
||||||
|
- module semantic ownership versus framework origin/aggregation ownership;
|
||||||
|
- bounded group and sample semantics;
|
||||||
|
- fresh/checkpoint equivalence; and
|
||||||
|
- the output-encoder decision.
|
||||||
|
|
||||||
|
Implementation should introduce `notarius.run-result.v2`. The grouped warning
|
||||||
|
and diagnostic envelopes should each carry their own schema version. Because
|
||||||
|
the content of `warnings.json` changes incompatibly from a flat array wrapper
|
||||||
|
to groups, release notes and the published JSON integration contract must call
|
||||||
|
out the migration. `index.json` gains the diagnostic file path.
|
||||||
|
|
||||||
|
Canonical documentation updates belong in:
|
||||||
|
|
||||||
|
- `docs/cli.md` for stderr presentation only;
|
||||||
|
- `docs/operations.md` for operator review and debug workflow;
|
||||||
|
- `docs/integrations/json-output.md` for warning and diagnostic file schemas;
|
||||||
|
- `docs/integrations/run-result.md` for v2 fields and compatibility;
|
||||||
|
- `docs/consumers/subprocess.md` and `docs/consumers/dnd-pipeline.md` for
|
||||||
|
downstream policy checks;
|
||||||
|
- `docs/internal/pipeline.md` for promotion, aggregation, retry, and checkpoint
|
||||||
|
mechanics;
|
||||||
|
- `docs/internal/modules.md` and `docs/internal/dnd.md` for producer rules and
|
||||||
|
the D&D classification matrix; and
|
||||||
|
- `docs/policy/architecture.md` for the durable ownership invariant after the
|
||||||
|
ADR is accepted and implemented.
|
||||||
|
|
||||||
|
No configuration knob is required for the first implementation. A fixed,
|
||||||
|
well-documented taxonomy is easier to reason about than per-reason display
|
||||||
|
overrides. Configurable escalation or suppression can be considered only after
|
||||||
|
production review demonstrates a concrete operator need.
|
||||||
|
|
||||||
|
## Test Coverage Assessment
|
||||||
|
|
||||||
|
Existing coverage worth preserving includes:
|
||||||
|
|
||||||
|
- accepted-attempt and terminal-rejection warning promotion;
|
||||||
|
- validator failure retry exhaustion and `warn_continue`;
|
||||||
|
- module semantic retry fallback;
|
||||||
|
- deterministic warning order under concurrent lane completion;
|
||||||
|
- chunk-plan invalidation and discarded-cache warning behavior;
|
||||||
|
- fresh/checkpoint warning equivalence;
|
||||||
|
- local D&D warning caps and safe dynamic-message quoting;
|
||||||
|
- JSON warning-file publication; and
|
||||||
|
- CLI stderr, debug, and receipt counts.
|
||||||
|
|
||||||
|
Material gaps are:
|
||||||
|
|
||||||
|
- no bound test for NPC- or spell-relatedness warnings;
|
||||||
|
- no generic warning-field or result-size validation;
|
||||||
|
- no test for output encoder warnings versus published `warnings.json`;
|
||||||
|
- no operator-level aggregation or bounded-sample tests;
|
||||||
|
- no fresh/resume test for grouped counts because groups do not yet exist; and
|
||||||
|
- no production evaluation of advisory precision.
|
||||||
|
|
||||||
|
Tests should protect the semantic relationships: exact occurrence counts,
|
||||||
|
bounded samples, deterministic group order, actionable-only warning
|
||||||
|
presentation, and cross-surface equality. They should not assert one exact
|
||||||
|
warning count for every complete D&D run or treat message wording as a public
|
||||||
|
API unless the wording itself enforces a security boundary.
|
||||||
|
|
||||||
|
## Audit Conclusion
|
||||||
|
|
||||||
|
The application is in a good position for warning reform. Its retry,
|
||||||
|
validation, checkpoint, and concurrency mechanics provide reliable points at
|
||||||
|
which to attach structured diagnostics. The most valuable change is not to
|
||||||
|
suppress individual reason codes; it is to replace the semantically flat,
|
||||||
|
origin-free collection with bounded typed groups and to reserve the word
|
||||||
|
“warning” for conditions that merit operator attention.
|
||||||
|
|
||||||
|
Provider-backed runs would improve prioritization and help tune the D&D
|
||||||
|
advisories, but they are not necessary to conclude that routine normalization
|
||||||
|
and heuristic doubt should not dominate stderr or the durable warning
|
||||||
|
contract. They should be gathered before changing heuristic acceptance policy
|
||||||
|
or adopting a numerical production warning-volume target.
|
||||||
@@ -1,516 +0,0 @@
|
|||||||
# Codebase Audit Plan
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document defines a repository-wide audit of Notarius for correctness,
|
|
||||||
efficiency, maintainability, and clarity. The audit should identify concrete
|
|
||||||
improvements without treating abstraction, fewer lines, or higher test coverage
|
|
||||||
as goals in themselves.
|
|
||||||
|
|
||||||
The audit is intentionally separate from implementation. Its findings should
|
|
||||||
be evidence-backed and sufficiently specific to support a later remediation
|
|
||||||
roadmap, but the audit should not modify production code, tests, assets, or
|
|
||||||
current-behavior documentation.
|
|
||||||
|
|
||||||
## Governing Principles
|
|
||||||
|
|
||||||
The audit must preserve the architecture and testing policies in
|
|
||||||
`docs/policy/architecture.md` and `docs/policy/testing.md`.
|
|
||||||
|
|
||||||
In particular:
|
|
||||||
|
|
||||||
- Notarius remains a fixed, staged pipeline rather than a general workflow
|
|
||||||
engine.
|
|
||||||
- Generic framework packages must remain domain-neutral, and production
|
|
||||||
modules must not acquire CLI or physical-state responsibilities.
|
|
||||||
- Typed artifact boundaries, exact codec compatibility, deterministic ordering,
|
|
||||||
whole-output validation, and generated-reference provenance are correctness
|
|
||||||
properties, not incidental complexity to be optimized away.
|
|
||||||
- The root `assets` package remains a content-only dependency leaf.
|
|
||||||
- Shared helpers should protect demonstrated common semantics. Similar-looking
|
|
||||||
code with different ownership, error policy, identity rules, or type contracts
|
|
||||||
should remain separate.
|
|
||||||
- Tests should protect durable behavior and meaningful risks. The audit should
|
|
||||||
not recommend tests merely to increase coverage or freeze implementation
|
|
||||||
details.
|
|
||||||
- Efficiency claims must distinguish measured or structurally credible costs
|
|
||||||
from cosmetic line-count reductions. Optimizing local CPU work that is
|
|
||||||
insignificant beside an LLM call is low priority unless it also simplifies
|
|
||||||
correctness or applies to large inputs.
|
|
||||||
|
|
||||||
## Audit Questions
|
|
||||||
|
|
||||||
Every audited area should be examined through the following questions.
|
|
||||||
|
|
||||||
### Correctness
|
|
||||||
|
|
||||||
- Are documented architecture invariants enforced at the correct boundary?
|
|
||||||
- Can invalid configuration, incompatible artifact types, malformed references,
|
|
||||||
or unavailable dependencies reach execution when they could be rejected
|
|
||||||
during resolution or preparation?
|
|
||||||
- Are nil, empty, absent, rejected, failed, and canceled states distinguished
|
|
||||||
consistently?
|
|
||||||
- Are stored or returned slices, maps, byte slices, options, metadata, source
|
|
||||||
documents, references, and artifacts defensively owned where required?
|
|
||||||
- Are public ordering, selected errors, warnings, and checkpoint decisions
|
|
||||||
deterministic regardless of map or goroutine completion order?
|
|
||||||
- Do cancellation, retry, validation, and partial-work semantics match their
|
|
||||||
documented ownership?
|
|
||||||
- Do checkpoint and chunk-plan identities include every semantic dependency and
|
|
||||||
exclude scheduling-only or diagnostic state?
|
|
||||||
- Can auxiliary references accidentally become source evidence, or can
|
|
||||||
generated references bypass codec, schema, provenance, or step-order checks?
|
|
||||||
- Can provider-specific values, credentials, or source content escape through
|
|
||||||
errors, manifests, debug summaries, cache state, or logs?
|
|
||||||
- Do schemas, codecs, candidate decoders, normalizers, and validators agree on
|
|
||||||
the exact durable contract without silently accepting incompatible shapes?
|
|
||||||
|
|
||||||
### Duplication And Shared Mechanics
|
|
||||||
|
|
||||||
- Which exact or near-duplicate implementations express the same invariant and
|
|
||||||
failure policy?
|
|
||||||
- Has duplicated code already drifted in naming, nil handling, canonicalization,
|
|
||||||
metadata, fingerprints, validation, or diagnostics?
|
|
||||||
- Would a helper have a natural owner and a smaller, clearer contract than the
|
|
||||||
duplicated callers?
|
|
||||||
- Can an extraction preserve static typing and package ownership, or would it
|
|
||||||
require reflection, `any`, callbacks with many policy parameters, or a
|
|
||||||
domain-neutral package importing domain concepts?
|
|
||||||
- Is repeated code required by a small interface adapter or typed registration
|
|
||||||
boundary and therefore clearer when left explicit?
|
|
||||||
|
|
||||||
As a default heuristic, prioritize a shared helper when identical semantics
|
|
||||||
appear in three or more production sites, or in two sites where divergence
|
|
||||||
would create a meaningful correctness risk. Do not use that heuristic as a
|
|
||||||
quota: one substantial duplicate may warrant extraction, while widespread
|
|
||||||
one-line interface methods may not.
|
|
||||||
|
|
||||||
### Simplicity And Idiomatic Go
|
|
||||||
|
|
||||||
- Does a function combine orchestration, policy, transformation, persistence,
|
|
||||||
and reporting that could be separated along existing ownership boundaries?
|
|
||||||
- Are repeated scans, sorts, conversions, clones, encodes, or decodes doing work
|
|
||||||
that can safely occur once?
|
|
||||||
- Are intermediate representations necessary, or can a value be validated,
|
|
||||||
canonicalized, and mapped in one comprehensible pass?
|
|
||||||
- Are maps, sets, stable sorts, generics, standard-library helpers, and error
|
|
||||||
wrapping used idiomatically?
|
|
||||||
- Are abstractions earning their complexity, or are interfaces, option layers,
|
|
||||||
wrappers, aliases, compatibility paths, and private types left over after a
|
|
||||||
completed migration?
|
|
||||||
- Are there unreachable error branches, redundant fingerprints or digests,
|
|
||||||
duplicated sources of truth, or accessors used only by tests?
|
|
||||||
- Can a smaller implementation preserve exact observable behavior and safety
|
|
||||||
properties?
|
|
||||||
|
|
||||||
### Explanatory Comments
|
|
||||||
|
|
||||||
Comments should be recommended where the code is necessarily complex because
|
|
||||||
it preserves a non-obvious invariant. Good candidates include:
|
|
||||||
|
|
||||||
- concurrency coordination, cancellation, and stable error selection;
|
|
||||||
- checkpoint identity, reuse, forced recomputation, and dependency invalidation;
|
|
||||||
- typed erasure and restoration at framework boundaries;
|
|
||||||
- generated-reference ordering and provenance;
|
|
||||||
- canonicalization and identity resolution where registry evidence differs
|
|
||||||
from occurrence evidence;
|
|
||||||
- prompt ordering or input identity required for backend caching; and
|
|
||||||
- path confinement, atomic publication, redaction, or terminal error precedence.
|
|
||||||
|
|
||||||
Recommend comments that explain *why* a step or ordering constraint exists and
|
|
||||||
what would break if it changed. Do not recommend comments that narrate syntax,
|
|
||||||
repeat a function name, duplicate current-behavior documentation, or preserve
|
|
||||||
implementation history.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
- Is each consequential invariant protected at the narrowest stable boundary?
|
|
||||||
- Are concurrency, cancellation, retries, recovery, compatibility, path safety,
|
|
||||||
and data-integrity behavior credibly exercised?
|
|
||||||
- Do higher-level contract tests duplicate lower-level cases without adding
|
|
||||||
integration confidence?
|
|
||||||
- Are tests coupled to private constants, helper shape, exact prose, full error
|
|
||||||
strings, or collaborator choreography rather than behavior?
|
|
||||||
- Can repetitive fixtures or fakes be simplified without creating a test
|
|
||||||
framework more complex than the tests?
|
|
||||||
- Would a focused fuzz test, race test, or package-level invariant test protect
|
|
||||||
a realistic risk better than several example tests?
|
|
||||||
|
|
||||||
## Evidence And Finding Standards
|
|
||||||
|
|
||||||
Static metrics and textual similarity are discovery aids, not findings. A long
|
|
||||||
function may be a clear linear coordinator; identical methods may be useful
|
|
||||||
typed adapters. Every reported finding must include:
|
|
||||||
|
|
||||||
1. a concise title and severity;
|
|
||||||
2. exact files and symbols;
|
|
||||||
3. the observed behavior or structural evidence;
|
|
||||||
4. the correctness, efficiency, maintenance, or comprehension impact;
|
|
||||||
5. a concrete recommended direction;
|
|
||||||
6. important invariants the remediation must preserve;
|
|
||||||
7. focused validation that would demonstrate success; and
|
|
||||||
8. whether the recommendation is independent or should be grouped with another
|
|
||||||
finding.
|
|
||||||
|
|
||||||
Use these severities:
|
|
||||||
|
|
||||||
- **High:** a credible risk of corrupt output, unsafe state handling, secret
|
|
||||||
exposure, stale reuse, deadlock, nondeterminism, or violated external
|
|
||||||
contract.
|
|
||||||
- **Medium:** a plausible behavioral defect, meaningful wasted work on common
|
|
||||||
paths, or complexity/duplication likely to cause future correctness drift.
|
|
||||||
- **Low:** a contained simplification, small efficiency improvement, dead code,
|
|
||||||
naming issue, or missing explanation with no current behavioral failure.
|
|
||||||
|
|
||||||
The audit should explicitly record examined areas with no findings. This makes
|
|
||||||
coverage visible and prevents later agents from repeatedly rediscovering the
|
|
||||||
same safe design.
|
|
||||||
|
|
||||||
## Repository Areas
|
|
||||||
|
|
||||||
### 1. Architecture And Dependency Boundaries
|
|
||||||
|
|
||||||
Inspect `docs/policy/architecture.md`, `docs/adr/`, `docs/internal/overview.md`,
|
|
||||||
package imports, module registrars, and the CLI composition root.
|
|
||||||
|
|
||||||
Look for:
|
|
||||||
|
|
||||||
- framework or core code depending on production modules;
|
|
||||||
- modules depending on CLI, physical roots, or provider-specific types;
|
|
||||||
- domain knowledge placed in generic helpers;
|
|
||||||
- duplicated registries or composition policy outside the owning registrar;
|
|
||||||
- abstractions that turn the fixed pipeline into an implicit general graph; and
|
|
||||||
- current code that no longer matches an accepted ADR or documented invariant.
|
|
||||||
|
|
||||||
Graph-reported cross-layer calls must be traced before being classified because
|
|
||||||
tests and interface implementations can resemble dependency inversions without
|
|
||||||
creating a production import violation.
|
|
||||||
|
|
||||||
### 2. Configuration And CLI Composition
|
|
||||||
|
|
||||||
Inspect `internal/core/config`, `internal/cli`, configuration parsing and
|
|
||||||
redaction tests, profile construction, session derivation, catalog assembly,
|
|
||||||
reference overrides, run-result handling, terminal reporting, and maintained
|
|
||||||
example contract tests.
|
|
||||||
|
|
||||||
Pay particular attention to the currently dense paths around
|
|
||||||
`runPipelineCommand`, configuration profile validation, selected reference
|
|
||||||
targets, recomputation policy, and option normalization. Determine whether
|
|
||||||
their complexity reflects necessary composition or mixed responsibilities that
|
|
||||||
can be separated without moving policy into the framework.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- file, environment, CLI, pipeline, binding, and prompt-default precedence;
|
|
||||||
- consistent strict option and unknown-field handling;
|
|
||||||
- session identity independence from references and pipeline-local changes;
|
|
||||||
- effective profile and runtime fingerprint consistency;
|
|
||||||
- redaction before errors or debug/manifest boundaries;
|
|
||||||
- output publication only after framework success; and
|
|
||||||
- one guarded terminalization path that preserves the primary failure.
|
|
||||||
|
|
||||||
### 3. Pipeline Resolution, Preparation, And Typed Registries
|
|
||||||
|
|
||||||
Inspect `internal/framework/pipeline/profile.go`, `prepare.go`, registry files,
|
|
||||||
`options.go`, `references.go`, `handoff.go`, `construction.go`, typed contracts,
|
|
||||||
and their focused tests.
|
|
||||||
|
|
||||||
This area deserves a dedicated pass because the current graph identifies
|
|
||||||
`ResolvePipeline`, generated-binding validation, reference-target resolution,
|
|
||||||
and generated-reference construction as high-complexity or high-fan-in code.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- static failures occur before source parsing;
|
|
||||||
- selected and unselected lanes do not contaminate each other's requirements;
|
|
||||||
- stage defaults and overrides have one canonical resolution path;
|
|
||||||
- typed registration and private erasure cannot panic or accept near-matching
|
|
||||||
artifact types;
|
|
||||||
- generated bindings reject cycles, forward references, ambiguity, wrong kinds,
|
|
||||||
and missing accepted normalized producers;
|
|
||||||
- materialized reference bytes and options are cloned and bounded; and
|
|
||||||
- resolved composition and prepared fingerprints include the complete semantic
|
|
||||||
policy exactly once.
|
|
||||||
|
|
||||||
Compare input, chunker, extractor, merger, normalizer, output, validator, codec,
|
|
||||||
evidence-projector, and validator-chain registries for shared mechanics and
|
|
||||||
intentional differences. Repeated typed registration code is a candidate only
|
|
||||||
if a helper can retain useful compile-time guarantees and stage-specific
|
|
||||||
diagnostics.
|
|
||||||
|
|
||||||
### 4. Pipeline Execution, Validation, Retry, And Concurrency
|
|
||||||
|
|
||||||
Inspect `runner*.go`, `typed_execution.go`, `runner_typed.go`,
|
|
||||||
`runner_concurrent.go`, validation-chain execution, normalize retry behavior,
|
|
||||||
synchronized collaborators, and the concurrency, cancellation, retry, debug,
|
|
||||||
and checkpoint tests.
|
|
||||||
|
|
||||||
Trace complete paths rather than reviewing helper files in isolation:
|
|
||||||
|
|
||||||
- source and chunk-plan selection through chunk validation;
|
|
||||||
- deterministic chunk-first/lane-second dispatch;
|
|
||||||
- lane extraction through merge and normalize continuations;
|
|
||||||
- rejection versus framework-error propagation;
|
|
||||||
- cancellation before dispatch, while queued, and while running;
|
|
||||||
- retry attempts and warning retention;
|
|
||||||
- stable error selection after concurrent completion;
|
|
||||||
- checkpoint hydration back into typed execution; and
|
|
||||||
- output suppression after a framework error.
|
|
||||||
|
|
||||||
Look for goroutine leaks, unbounded work, lock-order risks, double release or
|
|
||||||
double recording, races on shared result state, unnecessary serialization,
|
|
||||||
and repeated canonicalization. Comments are especially valuable here when they
|
|
||||||
explain ordering or cancellation invariants that are not apparent from local
|
|
||||||
control flow.
|
|
||||||
|
|
||||||
### 5. State, Checkpoints, Chunk Plans, Debugging, And File Safety
|
|
||||||
|
|
||||||
Inspect `internal/framework/checkpoint`, `chunkplan`, `chunkmap`, `debug`,
|
|
||||||
`evidencecontext`, `internal/core/fileio`, `debugbundle`, and their CLI
|
|
||||||
composition.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- narrow path validation and symlink-resistant confinement;
|
|
||||||
- atomic writes and recoverable explicit cleanup;
|
|
||||||
- separation of checkpoint recording, resume loading, chunk-plan caching, and
|
|
||||||
debug capture;
|
|
||||||
- canonical encoding before content identity is trusted;
|
|
||||||
- complete but non-secret checkpoint fingerprints;
|
|
||||||
- correct ordinary-resume and selective-recompute behavior;
|
|
||||||
- producer dependency invalidation across ordered steps;
|
|
||||||
- immutable hydration and no aliasing with stored bytes;
|
|
||||||
- debug data never influencing execution or reuse; and
|
|
||||||
- terminal persistence failures never obscuring the primary error.
|
|
||||||
|
|
||||||
Review the repeated extract/merge/normalize recorder and loader methods, path
|
|
||||||
component validators in multiple state packages, and clone/encode/decode paths.
|
|
||||||
Determine which repetition is a clear stage adapter and which can share a
|
|
||||||
private primitive without weakening reason-code ownership or diagnostics.
|
|
||||||
|
|
||||||
### 6. LLM Runtime, Prompt Filesystems, And Assets
|
|
||||||
|
|
||||||
Inspect `internal/framework/llm`, `promptfs`, the PromptKit integration,
|
|
||||||
scheduler, profile-source construction, prompt/schema registries, root
|
|
||||||
`assets`, module prompt manifests, and relevant D&D shared assets.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- every provider call passes through the shared scheduler and cancellation
|
|
||||||
removes queued calls safely;
|
|
||||||
- PromptKit and Notarius concurrency limits compose as documented;
|
|
||||||
- profile inspection and runtime use identical source precedence;
|
|
||||||
- session IDs, profile-source fingerprints, prompt fingerprints, and schema
|
|
||||||
fingerprints reflect the intended semantic inputs;
|
|
||||||
- secrets and provider-specific error types do not cross the boundary;
|
|
||||||
- prompt inputs and private outputs do not expose opaque entity IDs;
|
|
||||||
- prompt ordering, stable prefixes, and cache controls remain intentional;
|
|
||||||
- schema loaders and filesystem adapters validate once and return defensive
|
|
||||||
data; and
|
|
||||||
- the root assets package contains no business logic.
|
|
||||||
|
|
||||||
Compare the LLM asset registry and prompt-filesystem adapters for duplicated
|
|
||||||
filesystem behavior. Review repeated prompt/schema loader and metadata code in
|
|
||||||
module packages, but reject an extraction that would centralize domain prompt
|
|
||||||
ownership or make unrelated assets share one invalidation boundary.
|
|
||||||
|
|
||||||
### 7. Generic And Seriatim Modules
|
|
||||||
|
|
||||||
Inspect `internal/modules/generic` and `internal/modules/seriatim`, including
|
|
||||||
module specs, option decoding, chunk planning, input translation, validators,
|
|
||||||
output encoding, evidence-context publication, registration, and tests.
|
|
||||||
|
|
||||||
Verify that:
|
|
||||||
|
|
||||||
- external Seriatim details end at the input boundary;
|
|
||||||
- generic chunking and output remain domain-neutral;
|
|
||||||
- chunk plans and source units preserve source-addressed invariants;
|
|
||||||
- output logical names are safe and deterministic;
|
|
||||||
- output options do not bypass preparation-time compatibility checks; and
|
|
||||||
- option decoding is strict, small, and consistent with configuration
|
|
||||||
validation.
|
|
||||||
|
|
||||||
The graph flags generic integer option parsing and JSON output policy decoding
|
|
||||||
as relatively complex. Examine whether that is inherent strict decoding or an
|
|
||||||
opportunity for a smaller typed parser with equally precise diagnostics.
|
|
||||||
|
|
||||||
### 8. D&D Domain Model, Codecs, And Shared Helpers
|
|
||||||
|
|
||||||
Inspect `internal/modules/dnd` domain types, codecs, candidate decoders,
|
|
||||||
identity packages, registries, shared source-reference helpers, diagnostics,
|
|
||||||
registry resolution, entity reconciliation, mergers, registrar, and assets.
|
|
||||||
|
|
||||||
Compare all ten current artifact families. Build a convention matrix covering:
|
|
||||||
|
|
||||||
- module specs and execution classes;
|
|
||||||
- constructor and option behavior;
|
|
||||||
- manifest metadata and checkpoint fingerprints;
|
|
||||||
- response-schema loading and private-versus-durable types;
|
|
||||||
- source-reference conversion, canonicalization, ordering, and deduplication;
|
|
||||||
- nil versus present-empty output;
|
|
||||||
- codecs and strict JSON behavior;
|
|
||||||
- registry lookup, identity derivation, immutable projections, and resolution;
|
|
||||||
- normalizer retry/fallback behavior;
|
|
||||||
- validators and default chains; and
|
|
||||||
- registration, prompt assets, and documentation ownership.
|
|
||||||
|
|
||||||
The graph reports many exact similarities among codec `Decode` methods,
|
|
||||||
fingerprint/metadata methods, registry extractors, identity helpers, occurrence
|
|
||||||
normalizers, and validators. Treat these as a prioritized review list, not an
|
|
||||||
instruction to create one generic D&D engine. A worthwhile helper must preserve
|
|
||||||
domain-specific identity, evidence, kind ordering, validation, diagnostics,
|
|
||||||
and artifact typing.
|
|
||||||
|
|
||||||
### 9. D&D Extraction And Normalization Flows
|
|
||||||
|
|
||||||
Trace each lane end to end rather than auditing only similarly named files:
|
|
||||||
|
|
||||||
- spells;
|
|
||||||
- NPC registry and NPC occurrences;
|
|
||||||
- combat turns and enemy events;
|
|
||||||
- item registry and item occurrences;
|
|
||||||
- scene descriptions; and
|
|
||||||
- location registry and location occurrences.
|
|
||||||
|
|
||||||
For registry/occurrence pairs, verify the complete semantic boundary: the model
|
|
||||||
uses contextual evidence, deterministic code attaches opaque identity, registry
|
|
||||||
evidence does not become occurrence evidence, and unresolved or ambiguous
|
|
||||||
selections fail according to lane policy.
|
|
||||||
|
|
||||||
Review whether any lane resolves or canonicalizes the same entity, source
|
|
||||||
reference, or response twice; constructs unnecessary intermediate response
|
|
||||||
forms; performs repeated sorts or scans; or retains transitional paths. Compare
|
|
||||||
registry normalizers and occurrence normalizers for genuinely identical
|
|
||||||
mechanics, while keeping currency, same-name location, NPC ambiguity, spell
|
|
||||||
catalog, scene eligibility, and combat-specific policy with their owners.
|
|
||||||
|
|
||||||
### 10. Test Suite And Comment Coverage
|
|
||||||
|
|
||||||
Review the tests associated with every preceding area after understanding the
|
|
||||||
production contracts. This should be a cross-cutting pass, not a request to add
|
|
||||||
tests for every flagged function.
|
|
||||||
|
|
||||||
Identify:
|
|
||||||
|
|
||||||
- consequential unprotected invariants;
|
|
||||||
- duplicated policy assertions across layers;
|
|
||||||
- brittle tests coupled to internal constants, prompt prose, or private helper
|
|
||||||
shape;
|
|
||||||
- oversized test harnesses and repeated fixtures that obscure intent;
|
|
||||||
- race-sensitive code not exercised under `-race`;
|
|
||||||
- parsers, canonicalizers, and path handlers where fuzzing would address a real
|
|
||||||
input-space risk; and
|
|
||||||
- complex production code whose tests reveal an unclear ownership boundary.
|
|
||||||
|
|
||||||
Also identify necessarily complex symbols that lack a concise invariant-level
|
|
||||||
comment. Comment recommendations should name the exact symbol and the fact the
|
|
||||||
comment should explain; “add more comments” is not an actionable finding.
|
|
||||||
|
|
||||||
## Efficiency Evaluation
|
|
||||||
|
|
||||||
The audit should consider both runtime and maintenance efficiency.
|
|
||||||
|
|
||||||
For runtime efficiency, examine algorithmic behavior relative to realistic
|
|
||||||
input dimensions: source units, chunks, lanes, references, artifacts, registry
|
|
||||||
records, checkpoint files, and prompt assets. Prioritize repeated full-input
|
|
||||||
passes, nested linear lookup, unnecessary JSON round trips, repeated hashing,
|
|
||||||
large defensive copies at adjacent ownership boundaries, and serialization on
|
|
||||||
concurrent hot paths. Preserve a defensive copy when it establishes ownership;
|
|
||||||
removing it solely to reduce allocation is not an improvement.
|
|
||||||
|
|
||||||
For maintenance efficiency, prioritize repeated policy, parallel type systems,
|
|
||||||
duplicated error classification, scattered defaults, and migrations that left
|
|
||||||
two ways to perform the same operation. Boilerplate is costly only when it can
|
|
||||||
drift or obscures the semantic core. Small explicit typed adapters can be more
|
|
||||||
maintainable than a generic abstraction.
|
|
||||||
|
|
||||||
Do not recommend caching, pooling, concurrency, or a benchmark without naming
|
|
||||||
the workload and risk it addresses. Add a benchmark only when a proposed
|
|
||||||
optimization concerns a repeatable local path and the result would influence
|
|
||||||
the decision.
|
|
||||||
|
|
||||||
## Audit Method
|
|
||||||
|
|
||||||
Each area should use the same method:
|
|
||||||
|
|
||||||
1. Read its architecture/internal documentation and focused tests.
|
|
||||||
2. Map public/package contracts and trace the main call paths.
|
|
||||||
3. Inspect high-fan-in, high-cognitive-complexity, nested-loop, and repeated-
|
|
||||||
conversion symbols.
|
|
||||||
4. Review exact and near-duplicate code side by side, including callers and
|
|
||||||
failure semantics.
|
|
||||||
5. Check dependency direction, ownership, aliasing, deterministic order,
|
|
||||||
cancellation, and error classification.
|
|
||||||
6. Compare tests with the risks owned at that layer.
|
|
||||||
7. Record findings and inspected-with-no-finding areas before moving on.
|
|
||||||
8. Run focused read-only validation when it can confirm or refute a suspected
|
|
||||||
problem.
|
|
||||||
|
|
||||||
Prefer the repository knowledge graph for symbol discovery, call tracing, and
|
|
||||||
similarity candidates. Use textual search for literals, diagnostics, config
|
|
||||||
keys, asset content, and stale names. Read complete implementations and tests
|
|
||||||
before reporting a metric-derived candidate.
|
|
||||||
|
|
||||||
## Execution Sequence
|
|
||||||
|
|
||||||
This document owns audit scope, questions, evidence standards, and the quality
|
|
||||||
bar. [Staged Codebase Audit Sequence](audit-sequence.md) is the sole canonical
|
|
||||||
owner of prompt order, stage boundaries, per-stage reading, validation commands,
|
|
||||||
and acceptance criteria. Do not derive or maintain a second sequence here.
|
|
||||||
|
|
||||||
The audit is executed as bounded prompts and writes its accumulated findings to
|
|
||||||
`docs/roadmap/audit.md`. Later stages must build on and reconcile earlier
|
|
||||||
evidence rather than concatenate independent reports. Implementation and
|
|
||||||
roadmap retirement remain separate work after maintainers review the completed
|
|
||||||
audit.
|
|
||||||
|
|
||||||
## Baseline And Validation
|
|
||||||
|
|
||||||
Before the first audit stage, record the commit under review and require a clean
|
|
||||||
worktree. Refresh the code knowledge graph so renamed or deleted code does not
|
|
||||||
produce false findings. Run the normal offline baseline:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./cmd/notarius
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Run `go test -race` for packages with concurrency or mutable shared state,
|
|
||||||
especially `internal/framework/pipeline`, `internal/framework/llm`, state
|
|
||||||
packages, and D&D registries. A repository-wide race run is appropriate for
|
|
||||||
final verification if its cost remains reasonable.
|
|
||||||
|
|
||||||
Optional diagnostic commands should be used only when relevant:
|
|
||||||
|
|
||||||
- `go test -count=1` to rule out cache-masked failures;
|
|
||||||
- `go test -shuffle=on` to detect order coupling;
|
|
||||||
- focused fuzzing for existing or newly justified fuzz targets; and
|
|
||||||
- focused benchmarks or profiles for a specific efficiency finding.
|
|
||||||
|
|
||||||
The audit itself should not change tests to make the baseline pass. Record any
|
|
||||||
pre-existing failure and distinguish it from an audit finding.
|
|
||||||
|
|
||||||
## Deliverable Quality Bar
|
|
||||||
|
|
||||||
The completed audit should:
|
|
||||||
|
|
||||||
- cover every repository area listed above;
|
|
||||||
- distinguish defects from refactoring opportunities and comment requests;
|
|
||||||
- distinguish credible performance costs from aesthetic simplification;
|
|
||||||
- identify intentional duplication that should remain explicit;
|
|
||||||
- avoid recommendations that violate dependency direction or weaken typing;
|
|
||||||
- cite exact evidence and preserve named invariants for every finding;
|
|
||||||
- consolidate root causes rather than report many symptoms;
|
|
||||||
- rank independent work so a later implementation plan can stage it safely;
|
|
||||||
- recommend no code change whose expected benefit is smaller than its added
|
|
||||||
abstraction or test-maintenance cost; and
|
|
||||||
- leave implementation and roadmap retirement to later work.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None are required to begin the audit. If a later stage cannot determine whether
|
|
||||||
behavior is intentional from code, tests, policies, ADRs, or current
|
|
||||||
documentation, it should record the uncertainty and a recommended resolution
|
|
||||||
rather than silently treating preference as a defect.
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,350 +0,0 @@
|
|||||||
# Contextual Entity Grounding
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Notarius should use an LLM for semantic interpretation of source evidence, not
|
|
||||||
for referential-integrity work that deterministic code can perform more
|
|
||||||
reliably. D&D prompts must therefore stop requiring models to reproduce opaque
|
|
||||||
machine identifiers such as hash-derived entity IDs. Models should identify
|
|
||||||
entities through human-readable, evidence-grounded context, after which
|
|
||||||
Notarius resolves the selection and attaches the canonical durable identity.
|
|
||||||
|
|
||||||
This roadmap defines the policy, affected D&D prompt families, and intended
|
|
||||||
end state. The ordered work needed to reach that state is maintained in
|
|
||||||
[Implementation Plan](implementation.md).
|
|
||||||
|
|
||||||
## User Intent
|
|
||||||
|
|
||||||
The change has two goals:
|
|
||||||
|
|
||||||
- prevent otherwise useful model responses from failing because a long,
|
|
||||||
non-semantic string was copied incorrectly; and
|
|
||||||
- avoid spending prompt space and model effort on exact-copy work that provides
|
|
||||||
no semantic value.
|
|
||||||
|
|
||||||
The policy is not a ban on identifiers. Durable artifacts may continue to use
|
|
||||||
application-owned IDs, and prompts may continue to request source-unit ranges
|
|
||||||
that locate evidence. The policy governs which identity work is assigned to
|
|
||||||
the model.
|
|
||||||
|
|
||||||
## Policy
|
|
||||||
|
|
||||||
An LLM-facing prompt input or private response schema must not require a model
|
|
||||||
to reproduce an opaque machine identifier when Notarius can establish the same
|
|
||||||
association deterministically.
|
|
||||||
|
|
||||||
Opaque machine identifiers include cryptographic hashes, UUIDs, digests,
|
|
||||||
database keys, durable entity IDs, and other tokens whose characters do not
|
|
||||||
carry source-grounded meaning for the model. These values may remain in
|
|
||||||
application state, provenance, diagnostics, checkpoints, and durable artifact
|
|
||||||
contracts, but should be omitted from model-visible material when they do not
|
|
||||||
help the model make a semantic decision.
|
|
||||||
|
|
||||||
The intended responsibility boundary is:
|
|
||||||
|
|
||||||
- the model decides which contextual entity is supported by the supplied
|
|
||||||
evidence and returns the bounded semantic facts requested by the module;
|
|
||||||
- the calling module validates that the contextual selection resolves to
|
|
||||||
exactly one supplied candidate;
|
|
||||||
- deterministic code supplies the canonical display value and durable entity
|
|
||||||
ID; and
|
|
||||||
- existing validators continue to enforce referential integrity at later
|
|
||||||
artifact boundaries.
|
|
||||||
|
|
||||||
Transcript `start_unit_id` and `end_unit_id` values are permitted. They are
|
|
||||||
contextual source coordinates and form part of the evidence contract rather
|
|
||||||
than arbitrary identity tokens. Prompt IDs, schema IDs, fingerprints, session
|
|
||||||
IDs, and digests may also remain in runtime metadata that the model is not
|
|
||||||
asked to reproduce.
|
|
||||||
|
|
||||||
Short request-local labels are a narrowly permitted fallback only when a
|
|
||||||
contextual selector cannot uniquely represent the available choices without
|
|
||||||
unreasonable prompt cost. Such a label must be compact, scoped to one request,
|
|
||||||
validated against the supplied candidate set, and never reused as a durable
|
|
||||||
identity. Current D&D occurrence and reconciliation prompts should be designed
|
|
||||||
without this exception; adopting it later requires a concrete demonstrated
|
|
||||||
need and documented rationale.
|
|
||||||
|
|
||||||
## Current State
|
|
||||||
|
|
||||||
The initial NPC, item, and location registry extractors already follow the
|
|
||||||
desired pattern: the model returns contextual names and evidence, and Notarius
|
|
||||||
derives durable IDs afterward. Spells, combat turns, and enemy events use
|
|
||||||
contextual actor names rather than requiring hash-derived NPC IDs.
|
|
||||||
|
|
||||||
Two current prompt families diverge from that pattern:
|
|
||||||
|
|
||||||
1. `dnd/npc-occurrences`, `dnd/item-occurrences`, and
|
|
||||||
`dnd/location-occurrences` place durable registry IDs in model-visible
|
|
||||||
projections and require the private LLM response to repeat those IDs.
|
|
||||||
2. NPC-, item-, and location-registry normalization use the shared entity
|
|
||||||
reconciliation prompt, which labels candidates with opaque
|
|
||||||
`candidate-000001`-style keys and requires the model to copy those keys into
|
|
||||||
duplicate-group proposals.
|
|
||||||
|
|
||||||
The durable occurrence artifacts correctly retain canonical entity IDs. The
|
|
||||||
problem is the private model transport contract, not the published artifact
|
|
||||||
contract.
|
|
||||||
|
|
||||||
## Target Architecture
|
|
||||||
|
|
||||||
### Model proposals and durable artifacts
|
|
||||||
|
|
||||||
Private LLM response types must express contextual semantic proposals rather
|
|
||||||
than reuse the durable artifact type when that type contains an opaque entity
|
|
||||||
ID. The extractor maps a validated private response into the existing durable
|
|
||||||
artifact only after identity resolution succeeds.
|
|
||||||
|
|
||||||
No affected durable artifact kind, media type, schema ID, schema version, or
|
|
||||||
JSON field changes as part of this work. NPC, item, and location occurrence
|
|
||||||
artifacts continue to publish their exact canonical ID/name pair. Registry
|
|
||||||
artifacts likewise retain their IDs and evidence.
|
|
||||||
|
|
||||||
The private schemas and prompt declarations may remain at their current `v1`
|
|
||||||
identities because Notarius is pre-release and these are not external
|
|
||||||
contracts. Their content hashes, mapping-policy fingerprints, and affected
|
|
||||||
prompt fingerprints must change so incompatible checkpoints are not reused.
|
|
||||||
|
|
||||||
### NPC occurrence grounding
|
|
||||||
|
|
||||||
The NPC occurrence prompt receives an ordered names-only projection of the
|
|
||||||
normalized NPC registry. Its private response contains the canonical NPC name,
|
|
||||||
occurrence kind, and current-transcript source ranges, but no `npc_id`.
|
|
||||||
|
|
||||||
The extractor resolves the returned name under the existing NPC comparison
|
|
||||||
policy. Resolution must produce exactly one registry entry. It then writes that
|
|
||||||
entry's canonical display name and durable ID into the `dnd.NPCOccurrence`.
|
|
||||||
An unknown or ambiguous selection invalidates the extraction operation; the
|
|
||||||
extractor must not guess, use fuzzy matching, silently omit the record, or
|
|
||||||
accept a partial response.
|
|
||||||
|
|
||||||
### Item occurrence grounding
|
|
||||||
|
|
||||||
The item occurrence prompt receives an ordered names-only projection of the
|
|
||||||
normalized item registry. Its private response contains the canonical item
|
|
||||||
name, occurrence kind, kind-specific fields, and current-transcript source
|
|
||||||
ranges, but no `item_id`.
|
|
||||||
|
|
||||||
The extractor resolves the returned name under the existing item comparison
|
|
||||||
and identity policies. Resolution must produce exactly one registry entry,
|
|
||||||
whose canonical name and durable ID are attached deterministically. Unknown or
|
|
||||||
ambiguous selections invalidate the complete extraction operation rather than
|
|
||||||
being guessed, repaired by similarity, or dropped.
|
|
||||||
|
|
||||||
### Location occurrence grounding
|
|
||||||
|
|
||||||
Location identity cannot always be resolved from a display name alone: the
|
|
||||||
current registry intentionally permits same-name locations with distinct
|
|
||||||
source anchors. The location occurrence prompt must therefore receive a
|
|
||||||
contextual registry descriptor that contains the canonical display name plus
|
|
||||||
the minimum source-grounded registry evidence needed to distinguish same-name
|
|
||||||
records. It must not contain the durable `location:sha256:...` value.
|
|
||||||
|
|
||||||
The private response uses two required selector fields: `name` and
|
|
||||||
`registry_refs`. For a comparison-unique canonical name, `registry_refs` is an
|
|
||||||
empty array and Notarius resolves the name under the location comparison
|
|
||||||
policy. For a name shared by multiple registry records, `registry_refs`
|
|
||||||
contains that record's complete canonically ordered registry ranges as
|
|
||||||
`start_unit_id` and `end_unit_id` pairs, without `source_id`.
|
|
||||||
|
|
||||||
Every model-facing registry entry uses one fixed shape with required `name`,
|
|
||||||
`registry_refs`, and `context` fields. `context` is an array of strict objects
|
|
||||||
containing only `unit_id` and `text`. Comparison-unique entries use empty
|
|
||||||
`registry_refs` and `context` arrays. Same-name entries use the complete
|
|
||||||
registry-range selector and the bounded context described below. The model
|
|
||||||
returns only `name` and `registry_refs`; it does not reproduce `context`.
|
|
||||||
|
|
||||||
For same-name groups, the projection also supplies bounded transcript units
|
|
||||||
covered by each record's registry ranges so the model receives meaningful
|
|
||||||
identity context rather than coordinates alone. Those ranges must resolve
|
|
||||||
against the current source document, and the resulting contextual selectors
|
|
||||||
must be unique. An invalid range or selector collision prevents the LLM call
|
|
||||||
and fails the operation. Unique-name entries do not repeat registry ranges or
|
|
||||||
context in the selector, preserving compatibility with a valid registry from
|
|
||||||
another source when the name alone is unambiguous.
|
|
||||||
|
|
||||||
The private response separately supplies current-transcript `source_refs` that
|
|
||||||
prove the occurrence. Registry identity evidence and occurrence evidence must
|
|
||||||
remain different fields and must never be merged. The model should omit an
|
|
||||||
occurrence when the transcript does not support choosing among same-name
|
|
||||||
locations. If a returned selector does not resolve to exactly one supplied
|
|
||||||
registry record, the extractor invalidates the complete operation rather than
|
|
||||||
guessing.
|
|
||||||
|
|
||||||
### Registry reconciliation
|
|
||||||
|
|
||||||
The shared entity-reconciliation input replaces opaque candidate keys with
|
|
||||||
contextual candidate descriptors. At minimum, a descriptor contains the
|
|
||||||
candidate's display name and its canonically ordered source-reference ranges;
|
|
||||||
the existing transcript windows remain available for semantic judgment.
|
|
||||||
|
|
||||||
Duplicate-group members and the canonical member in the private response use
|
|
||||||
the same contextual descriptor shape. The shared reconciliation helper maps
|
|
||||||
each descriptor back to exactly one internal candidate before assessing the
|
|
||||||
proposal. Exact deterministic duplicates should already be removed before the
|
|
||||||
LLM call; any remaining descriptor collision makes the affected candidate
|
|
||||||
ineligible for model-assisted reconciliation rather than authorizing an
|
|
||||||
arbitrary choice.
|
|
||||||
|
|
||||||
Existing safety behavior remains in force: groups must contain at least two
|
|
||||||
supplied candidates, the canonical candidate must be a member, groups must not
|
|
||||||
overlap, and domain-specific eligibility rules remain authoritative. Invalid,
|
|
||||||
ambiguous, or unsafe groups are discarded through the existing bounded
|
|
||||||
fallback and diagnostic behavior. The model never directly mutates the
|
|
||||||
durable registry.
|
|
||||||
|
|
||||||
The shared private reconciliation schema and helper must remain domain-neutral
|
|
||||||
within the D&D family. NPC-, item-, and location-specific duplicate policy
|
|
||||||
continues to live in the owning normalizer.
|
|
||||||
|
|
||||||
## Prompt And Asset Changes
|
|
||||||
|
|
||||||
The following LLM-facing assets are in scope:
|
|
||||||
|
|
||||||
- the prompt instructions, registry input fragments, and private response
|
|
||||||
schemas for NPC, item, and location occurrences;
|
|
||||||
- the prompt manifests where input shape or selected fragments change;
|
|
||||||
- the shared D&D entity-reconciliation fragment and private response schema;
|
|
||||||
and
|
|
||||||
- the NPC-, item-, and location-registry normalization prompt inputs that use
|
|
||||||
the shared reconciliation contract.
|
|
||||||
|
|
||||||
Affected projections must exclude durable entity IDs rather than merely stop
|
|
||||||
mentioning them in prose. Prompt instructions should describe the contextual
|
|
||||||
selection rule once at the narrowest owning asset and must preserve the current
|
|
||||||
distinction between registry grounding and transcript evidence.
|
|
||||||
|
|
||||||
Prompt ordering and cache controls should remain unchanged unless the new
|
|
||||||
contextual input requires an intentional manifest change. Unrelated shared
|
|
||||||
prompt bytes should not be edited. Prompt and schema fingerprints should
|
|
||||||
invalidate only the operations whose selected assets or mapping semantics
|
|
||||||
changed.
|
|
||||||
|
|
||||||
## Code And Validation Changes
|
|
||||||
|
|
||||||
The occurrence extractors need private response types and deterministic
|
|
||||||
registry-resolution paths appropriate to their domain. Shared code is
|
|
||||||
appropriate only for demonstrated mechanics that have identical semantics;
|
|
||||||
NPC, item, and location ambiguity policies must not be forced behind a generic
|
|
||||||
resolver merely to reduce line count.
|
|
||||||
|
|
||||||
Registry projections should expose explicit model-facing methods whose names
|
|
||||||
describe whether they are names-only or contextual identity projections. The
|
|
||||||
existing ID/name projections may remain only for deterministic consumers that
|
|
||||||
genuinely require them; they must no longer be wired to an LLM input.
|
|
||||||
|
|
||||||
Mapping-policy and normalization-policy identifiers must be reviewed and
|
|
||||||
advanced wherever their semantics change. Checkpoint fingerprints must cover
|
|
||||||
the new projection content, private schema, prompt assets, and mapping policy,
|
|
||||||
while continuing to exclude irrelevant internal implementation details.
|
|
||||||
|
|
||||||
Durable occurrence normalizers and registry validators remain defense in
|
|
||||||
depth. They continue to validate exact ID/name pairs on artifacts entering
|
|
||||||
through checkpoints, codecs, or other boundaries even though the LLM no longer
|
|
||||||
produces the ID directly.
|
|
||||||
|
|
||||||
## Testing And Evaluation
|
|
||||||
|
|
||||||
Tests should protect the behavioral boundary rather than prompt prose or
|
|
||||||
private helper structure. The completed work should demonstrate that:
|
|
||||||
|
|
||||||
- affected model-facing registry projections do not contain durable entity
|
|
||||||
IDs;
|
|
||||||
- private occurrence schemas reject opaque ID fields and accept the intended
|
|
||||||
contextual shape;
|
|
||||||
- valid contextual selections map to the exact canonical durable ID/name pair;
|
|
||||||
- unknown, mismatched, and ambiguous selections fail without fuzzy matching,
|
|
||||||
partial acceptance, or arbitrary reassignment;
|
|
||||||
- same-name locations remain distinguishable through contextual evidence;
|
|
||||||
- reconciliation preserves equal-name candidates, resolves valid contextual
|
|
||||||
groups, and discards ambiguous or unsafe proposals;
|
|
||||||
- registry evidence never becomes occurrence evidence;
|
|
||||||
- durable codec, normalization, and validator behavior remains compatible; and
|
|
||||||
- representative assembled D&D pipelines still prepare and execute with fake
|
|
||||||
structured-LLM responses.
|
|
||||||
|
|
||||||
Do not add repository-wide prompt-prose snapshots, exact-message-count tests,
|
|
||||||
or a change-detector test that merely scans for today's field names. Focused
|
|
||||||
projection, schema, mapping, fallback, and integration tests are the stable
|
|
||||||
owners of these risks. Model-quality evaluation with representative
|
|
||||||
transcripts remains a manual development aid rather than an offline test gate.
|
|
||||||
|
|
||||||
## Documentation And Architectural Record
|
|
||||||
|
|
||||||
This policy is durable and applies to future modules, so it warrants
|
|
||||||
`docs/adr/0012-resolve-opaque-entity-identifiers-deterministically.md`, which
|
|
||||||
records:
|
|
||||||
|
|
||||||
- the semantic-proposal versus referential-integrity boundary;
|
|
||||||
- why durable opaque IDs are excluded from model response contracts;
|
|
||||||
- why contextual evidence coordinates remain permitted;
|
|
||||||
- the narrowly scoped request-local-label exception;
|
|
||||||
- alternatives including durable IDs, names-only matching, and short opaque
|
|
||||||
handles; and
|
|
||||||
- the consequences for private schemas, deterministic resolution, debugging,
|
|
||||||
and ambiguous identities.
|
|
||||||
|
|
||||||
`docs/policy/architecture.md` states the general LLM boundary invariant and
|
|
||||||
links to the ADR. `docs/internal/dnd.md` describes the concrete occurrence
|
|
||||||
projections, contextual reconciliation selectors, resolution and failure
|
|
||||||
behavior, and the continued separation of registry grounding from occurrence
|
|
||||||
evidence. `docs/internal/llm.md` contains only a short clarification that
|
|
||||||
caller-owned modules, not PromptKit or the transport adapter, resolve
|
|
||||||
contextual model selections into application identities.
|
|
||||||
|
|
||||||
The NPC, item, and location occurrence and registry integration documents must
|
|
||||||
continue to own their durable wire contracts, while removing current claims
|
|
||||||
that the model-facing consumer projection contains `{id,name}` or that the raw
|
|
||||||
LLM response supplies the durable ID. They should instead explain that
|
|
||||||
Notarius resolves contextual model output and publishes the same exact durable
|
|
||||||
ID/name pair. No public schema examples need to remove those IDs.
|
|
||||||
|
|
||||||
The generic LLM-assisted deduplication entry in `docs/roadmap/future.md` must be
|
|
||||||
reconciled with this policy: stable IDs may exist inside deterministic state,
|
|
||||||
but a future model-facing proposal should use contextual selectors or a
|
|
||||||
documented request-local-label exception rather than durable IDs.
|
|
||||||
|
|
||||||
## Compatibility And Operational Effects
|
|
||||||
|
|
||||||
This work intentionally changes private prompt inputs, private structured
|
|
||||||
responses, and mapping semantics. It will invalidate affected checkpoints
|
|
||||||
through existing prompt, schema, projection, and policy fingerprints. No
|
|
||||||
manual checkpoint migration is required.
|
|
||||||
|
|
||||||
Durable D&D artifacts and generated-reference compatibility remain unchanged.
|
|
||||||
Operators do not receive new configuration fields or CLI controls. The feature
|
|
||||||
does not change PromptKit, provider routing, profile selection, retries,
|
|
||||||
concurrency, or public output placement.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This work does not:
|
|
||||||
|
|
||||||
- remove canonical IDs from durable registries or occurrence artifacts;
|
|
||||||
- change occurrence categories, evidence rules, or registry identity policy;
|
|
||||||
- add fuzzy, probabilistic, or embedding-based entity resolution;
|
|
||||||
- allow registry provenance to substitute for occurrence evidence;
|
|
||||||
- introduce a general entity graph or cross-artifact identity framework;
|
|
||||||
- redesign unrelated D&D prompts or their schemas;
|
|
||||||
- implement the future generic deduplication normalizer; or
|
|
||||||
- add provider-specific prompt behavior.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
The target state is complete when:
|
|
||||||
|
|
||||||
- no maintained D&D prompt requires a model to reproduce a durable opaque
|
|
||||||
entity ID;
|
|
||||||
- current D&D reconciliation prompts no longer require opaque candidate keys;
|
|
||||||
- NPC, item, and location occurrence LLM outputs are resolved
|
|
||||||
deterministically into their unchanged durable artifacts;
|
|
||||||
- same-name location and reconciliation cases remain safe and unambiguous;
|
|
||||||
- invalid contextual selections preserve the existing extraction-failure or
|
|
||||||
normalization-fallback semantics appropriate to their stage;
|
|
||||||
- affected checkpoint identities change without altering public schema
|
|
||||||
versions;
|
|
||||||
- focused and repository-wide tests pass offline;
|
|
||||||
- the ADR, architecture invariant, D&D internal guide, LLM internal guide,
|
|
||||||
relevant integration contracts, and future roadmap accurately describe
|
|
||||||
their canonical portions of the implemented policy; and
|
|
||||||
- no unrelated code, prompt behavior, or public contract changes are included.
|
|
||||||
@@ -5,13 +5,64 @@ configuration, operations, internal, and integration docs. This roadmap records
|
|||||||
future work only. Items are ordered roughly by current value and specificity,
|
future work only. Items are ordered roughly by current value and specificity,
|
||||||
not as committed release dates.
|
not as committed release dates.
|
||||||
|
|
||||||
|
## Near-Term Validation And LLM Reliability
|
||||||
|
|
||||||
|
PromptKit now owns structural output repair within one completion. Notarius
|
||||||
|
owns stage candidates, validator chains, semantic rejection policy, bounded
|
||||||
|
feedback-aware stage retries, validation provenance, and reusable-state
|
||||||
|
eligibility, and the separation of actionable process warnings from quality
|
||||||
|
diagnostics. The remaining near-term work applies those completed foundations
|
||||||
|
to domain review and empirical evaluation.
|
||||||
|
|
||||||
|
### D&D Combat Scene Semantic Validation
|
||||||
|
|
||||||
|
- Add an optional production LLM-backed D&D validator that determines whether
|
||||||
|
proposed scene boundaries and classifications represent substantive active
|
||||||
|
combat correctly. Its central quality goal is that active combat is kept in
|
||||||
|
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||||
|
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||||
|
- Resolve the validator's exact target before implementation. The current
|
||||||
|
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||||
|
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||||
|
`recap`, and `meta` classification. The preferred initial placement is
|
||||||
|
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||||
|
can compare one proposed kind with the corresponding transcript chunk.
|
||||||
|
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||||
|
question that can be answered from the complete transcript and proposed
|
||||||
|
range map, such as whether one continuous combat was fragmented across
|
||||||
|
inappropriate scene boundaries. Do not duplicate the same classification
|
||||||
|
judgment at both stages. Moving classification into chunk-plan annotations
|
||||||
|
would change the deliberately minimal, annotation-free chunk contract and
|
||||||
|
requires an explicit architecture review before it is selected.
|
||||||
|
- Validate both false negatives and false positives: a non-combat kind must not
|
||||||
|
omit substantive active combat, and a combat kind must be supported by such
|
||||||
|
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||||
|
extraction runs only for an exact `combat` scene classification; semantic
|
||||||
|
review improves the upstream classification but does not replace that gate.
|
||||||
|
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||||
|
structured response schema, and let PromptKit repair structural validator
|
||||||
|
output within its bounded budget. A contract-invalid final validator response
|
||||||
|
is a validator execution failure, not a semantic rejection and not a reason
|
||||||
|
to recursively validate the validator.
|
||||||
|
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||||
|
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||||
|
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||||
|
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||||
|
added calls, latency, and token cost before placing it in the production
|
||||||
|
default chain.
|
||||||
|
- An ADR is not required if classification remains owned by
|
||||||
|
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||||
|
Create or supersede an ADR if the work transfers scene classification into
|
||||||
|
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||||
|
contract.
|
||||||
|
|
||||||
## Near-Term D&D Pipeline
|
## Near-Term D&D Pipeline
|
||||||
|
|
||||||
### Evaluate Spell Extraction And Normalization
|
### Evaluate Spell Extraction And Normalization
|
||||||
|
|
||||||
- Evaluate ordinary extraction retries and the completed normalization path
|
- Evaluate ordinary extraction retries and the completed normalization path
|
||||||
against a human-reviewed transcript set before adding repair-aware retries or
|
against a human-reviewed transcript set before and after adopting the shared
|
||||||
an LLM-backed semantic validator.
|
PromptKit repair and Notarius validation-retry policies above.
|
||||||
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
||||||
validator, and normalizer development. Treat model-quality review as an
|
validator, and normalizer development. Treat model-quality review as an
|
||||||
iterative human evaluation aid, not a deterministic correctness gate.
|
iterative human evaluation aid, not a deterministic correctness gate.
|
||||||
@@ -24,28 +75,51 @@ not as committed release dates.
|
|||||||
|
|
||||||
## Shared Normalization And Quality Work
|
## Shared Normalization And Quality Work
|
||||||
|
|
||||||
### Generic LLM-Assisted Deduplication
|
The implemented source-backed core and initial D&D registry adoption are
|
||||||
|
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
|
||||||
|
and
|
||||||
|
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||||
|
The sections below keep broader extensions deferred.
|
||||||
|
|
||||||
- Add a reusable normalizer that asks an LLM to identify duplicate sets in a
|
### Large-Collection Semantic Reconciliation
|
||||||
list and propose one replacement element for each set.
|
|
||||||
- Define the minimum domain-neutral input contract, initially an ordered list
|
|
||||||
whose elements retain stable unique IDs as internal deterministic state.
|
|
||||||
Model proposals use contextual descriptors, or a specifically justified
|
|
||||||
request-local short label, rather than durable IDs. Artifact-kind
|
|
||||||
registrations or adapters may expose that structure without moving domain
|
|
||||||
rules into the generic package.
|
|
||||||
- Keep mutation deterministic: parse and validate the model's duplicate groups,
|
|
||||||
resolve every supplied descriptor or local label exactly, reject overlapping
|
|
||||||
or malformed groups, prevent unrelated insertion or deletion, and apply only
|
|
||||||
approved replacement operations in code.
|
|
||||||
- Preserve provenance needed for audit and downstream validation, and emit
|
|
||||||
warnings describing every collapsed group.
|
|
||||||
- Evaluate batching and context-window limits before applying the normalizer to
|
|
||||||
large artifact collections.
|
|
||||||
|
|
||||||
The model may use its own domain knowledge to judge semantic duplication; the
|
- Evaluate deterministic candidate blocking only after representative registry
|
||||||
generic implementation is responsible only for the common proposal contract,
|
inputs exceed the active roadmap's bounded single-request limits. Blocking
|
||||||
safety checks, and deterministic application of accepted changes.
|
should use cheap, explainable signals to form plausible comparison sets while
|
||||||
|
preserving the possibility that a duplicate appears outside a lexical name
|
||||||
|
match.
|
||||||
|
- Define correctness for candidates that appear in more than one block,
|
||||||
|
conflicting canonical selections, transitive identity across blocks, retry
|
||||||
|
isolation, and deterministic final ordering before implementation.
|
||||||
|
- Prefer a reconciliation graph or union plan with explicit conflict checks
|
||||||
|
over arbitrary fixed-size slices. Never silently treat a batch boundary as
|
||||||
|
evidence that two candidates are distinct.
|
||||||
|
- Record per-request bounds, block provenance, model calls, discarded
|
||||||
|
proposals, and final group derivation well enough to audit a collapse.
|
||||||
|
|
||||||
|
### Operator-Selected Semantic Policies
|
||||||
|
|
||||||
|
- Consider allowing an operator to select an approved semantic-policy prompt
|
||||||
|
for a typed reconciliation module without replacing the shared protocol,
|
||||||
|
response schema, or deterministic safety rules.
|
||||||
|
- Define the trusted asset source, configuration syntax, compatibility checks,
|
||||||
|
startup validation, provenance, prompt fingerprinting, checkpoint effects,
|
||||||
|
and support boundary before exposing the option.
|
||||||
|
- Prefer selection among registered, typed-policy-compatible prompt assets over
|
||||||
|
arbitrary filesystem prompt paths. Do not add this flexibility until an
|
||||||
|
operator workflow requires it; artifact-family-owned policy remains simpler
|
||||||
|
and safer for the initial implementation.
|
||||||
|
|
||||||
|
### Broader Reconciliation Inputs And Module Selection
|
||||||
|
|
||||||
|
- Revisit alternate context providers when a concrete non-source-backed entity
|
||||||
|
collection needs semantic reconciliation. Any extension must preserve the
|
||||||
|
same request-local identity, deterministic proposal validation, provenance,
|
||||||
|
and typed application guarantees.
|
||||||
|
- Consider a selectable generic normalizer only if Notarius gains a real
|
||||||
|
domain-neutral typed artifact contract that can safely support it. Do not
|
||||||
|
weaken exact artifact registration or introduce reflection-based arbitrary
|
||||||
|
JSON mutation merely to expose a universal module key.
|
||||||
|
|
||||||
### Validation And Review
|
### Validation And Review
|
||||||
|
|
||||||
@@ -102,6 +176,18 @@ checkpoint reuse, when an older artifact may be decoded or adapted, and when a
|
|||||||
producer or all dependents must be recomputed. Do not add a general migration
|
producer or all dependents must be recomputed. Do not add a general migration
|
||||||
framework until an actual contract change requires one.
|
framework until an actual contract change requires one.
|
||||||
|
|
||||||
|
### Artifact-family-oriented physical packaging
|
||||||
|
|
||||||
|
[ADR-0004](../adr/0004-package-modules-by-domain.md) currently groups production
|
||||||
|
extensions by domain and then by pipeline stage. After artifact-family
|
||||||
|
ownership terminology is established and more families span extraction,
|
||||||
|
normalization, validation, codecs, references, and assets, reassess whether a
|
||||||
|
feature-first physical layout would improve navigation and reduce scattered
|
||||||
|
changes enough to justify a repository-wide package migration. Any change must
|
||||||
|
address Go dependency cycles, registrar ownership, stable public module keys,
|
||||||
|
and supersession of the affected ADR-0004 decision. Conceptual artifact-family
|
||||||
|
ownership does not by itself require this move.
|
||||||
|
|
||||||
## Blue-Sky Platform And Operations
|
## Blue-Sky Platform And Operations
|
||||||
|
|
||||||
These ideas are intentionally less specified. Promote one into an earlier
|
These ideas are intentionally less specified. Promote one into an earlier
|
||||||
@@ -117,7 +203,6 @@ section only after a concrete workflow, contract, and priority emerge.
|
|||||||
### Distribution And Operations
|
### Distribution And Operations
|
||||||
|
|
||||||
- Packaged release artifacts for alpha distribution.
|
- Packaged release artifacts for alpha distribution.
|
||||||
- A documented versioning and release process.
|
|
||||||
- Optional generated example-output fixtures with a regeneration procedure.
|
- Optional generated example-output fixtures with a regeneration procedure.
|
||||||
- Additional diagnostics or reporting views.
|
- Additional diagnostics or reporting views.
|
||||||
|
|
||||||
|
|||||||
@@ -1,612 +0,0 @@
|
|||||||
# Contextual Entity Grounding Implementation Plan
|
|
||||||
|
|
||||||
## Objective
|
|
||||||
|
|
||||||
Implement [Contextual Entity Grounding](contextual-entity-grounding.md) so D&D
|
|
||||||
LLM prompts return evidence-grounded contextual selectors while Notarius owns
|
|
||||||
canonical entity IDs and referential integrity. Preserve every durable D&D
|
|
||||||
artifact contract and remove opaque IDs only from model-visible inputs and
|
|
||||||
private model responses.
|
|
||||||
|
|
||||||
This plan is written for a gpt-5.6-terra coding agent. Implement the stages in
|
|
||||||
numeric order. Each stage is intentionally scoped to one implementation prompt
|
|
||||||
and must leave the repository buildable and its focused tests passing before
|
|
||||||
the next stage begins.
|
|
||||||
|
|
||||||
## Plan-Wide Decisions
|
|
||||||
|
|
||||||
Apply these decisions throughout every stage:
|
|
||||||
|
|
||||||
- Read `docs/development.md`, all files under `docs/policy/`, the feature
|
|
||||||
roadmap, and the focused implementation/tests named by the stage before
|
|
||||||
editing.
|
|
||||||
- Preserve the fixed pipeline, typed artifact boundaries, root `assets`
|
|
||||||
content-only rule, module ownership, PromptKit boundary, and evidence rules.
|
|
||||||
- Do not change the durable NPC-, item-, location-registry, or occurrence Go
|
|
||||||
types, JSON schemas, schema IDs, schema versions, media types, reference-slot
|
|
||||||
contracts, categories, or generated-reference compatibility.
|
|
||||||
- Keep the affected prompt and private response-schema identities at `v1`.
|
|
||||||
They are private pre-release transport contracts; their changed content
|
|
||||||
hashes provide the required compatibility boundary.
|
|
||||||
- Advance semantic policy identifiers exactly as directed in each stage. Do
|
|
||||||
not bump unrelated policy identifiers.
|
|
||||||
- A contextual name match uses the entity family's existing comparison policy,
|
|
||||||
never fuzzy matching. A model selection must resolve to exactly one supplied
|
|
||||||
record before a durable ID is attached.
|
|
||||||
- An invalid NPC, item, or location selection invalidates the complete
|
|
||||||
extraction operation. Do not silently drop one response record, accept a
|
|
||||||
partial artifact, or defer a known mapping failure to a later validator.
|
|
||||||
- Registry provenance remains grounding only. Only the current extraction
|
|
||||||
chunk's `source_refs` become occurrence evidence.
|
|
||||||
- Preserve prompt message order and cache controls unless a stage explicitly
|
|
||||||
directs otherwise. Edit only the selected module or shared assets; do not
|
|
||||||
rewrite unrelated shared prompt bytes.
|
|
||||||
- Preserve internal opaque IDs where deterministic code needs them. The rule
|
|
||||||
applies to material shown to the model or requested from it, not to maps,
|
|
||||||
fingerprints, checkpoints, diagnostics, or durable artifacts.
|
|
||||||
- Follow `docs/policy/testing.md`: test package-level behavior and meaningful
|
|
||||||
failure modes, not prompt prose, exact message counts, private helper
|
|
||||||
structure, or a repository-wide string-scanning change detector. All tests
|
|
||||||
remain deterministic, offline, and credential-free.
|
|
||||||
- Use `apply_patch` for edits, `gofmt` changed Go files, and preserve unrelated
|
|
||||||
worktree changes.
|
|
||||||
|
|
||||||
## Final Private Selector Contracts
|
|
||||||
|
|
||||||
These shapes are implementation requirements, not public artifact schemas.
|
|
||||||
|
|
||||||
### NPC occurrence response
|
|
||||||
|
|
||||||
Each response record contains exactly the required fields `name`, `kind`, and
|
|
||||||
`source_refs`. It does not contain `npc_id`. Notarius resolves `name` through
|
|
||||||
the normalized NPC registry and writes the matched registry record's `ID` and
|
|
||||||
canonical `Name` into the durable occurrence.
|
|
||||||
|
|
||||||
### Item occurrence response
|
|
||||||
|
|
||||||
Each response record contains the existing required `name`, `kind`,
|
|
||||||
`quantity`, `from`, `to`, and `source_refs` fields. It does not contain
|
|
||||||
`item_id`. Retain the current nullable representation and kind-specific
|
|
||||||
semantics. Notarius resolves `name` through the normalized item registry and
|
|
||||||
adds the matched `ID` and canonical `Name`.
|
|
||||||
|
|
||||||
### Location occurrence response
|
|
||||||
|
|
||||||
Each response record contains exactly the required fields `name`,
|
|
||||||
`registry_refs`, `kind`, and `source_refs`. `registry_refs` is always an array
|
|
||||||
of strict objects containing required integer `start_unit_id` and
|
|
||||||
`end_unit_id`; it may be empty.
|
|
||||||
|
|
||||||
- When `name` has one comparison-identity match in the supplied registry,
|
|
||||||
`registry_refs` must be empty and name resolution selects that record.
|
|
||||||
- When multiple registry records share the comparison identity,
|
|
||||||
`registry_refs` must equal one record's complete canonically ordered source
|
|
||||||
ranges with `source_id` removed.
|
|
||||||
- The model-facing registry projection uses the same `name` plus
|
|
||||||
`registry_refs` selector and adds a required `context` array. Unique-name
|
|
||||||
records project empty `registry_refs` and `context` arrays. The private
|
|
||||||
response does not reproduce `context`.
|
|
||||||
- Same-name records receive bounded identity context consisting of the ordered
|
|
||||||
source units covered by their registry ranges. Each context element is a
|
|
||||||
strict object with exactly the required fields `unit_id` (integer) and
|
|
||||||
`text` (string); do not expose the durable location ID, source ID, digest,
|
|
||||||
or a replacement token.
|
|
||||||
- Building same-name grounding validates that every registry range belongs to
|
|
||||||
and resolves against the current source. If two records still produce the
|
|
||||||
same contextual selector, grounding construction fails before the LLM call.
|
|
||||||
- `registry_refs` never flow into the durable occurrence's `source_refs`.
|
|
||||||
|
|
||||||
### Entity-reconciliation response
|
|
||||||
|
|
||||||
The shared response remains an object with required `duplicate_groups`.
|
|
||||||
Every group has required `members` and `canonical`. A member and the canonical
|
|
||||||
selection are strict contextual objects containing:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "Mira Thorn",
|
|
||||||
"source_refs": [
|
|
||||||
{"start_unit_id": 12, "end_unit_id": 12}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The candidate prompt input uses the same descriptor and contains no `key`.
|
|
||||||
`source_refs` is required and non-empty for every eligible candidate. The
|
|
||||||
shared helper may retain its existing `candidate-000001`-style keys strictly
|
|
||||||
inside Go state to preserve input-position mapping; those keys must never be
|
|
||||||
serialized into prompt input or accepted in the private response.
|
|
||||||
|
|
||||||
If two candidates produce an identical contextual descriptor, neither is
|
|
||||||
eligible for model-assisted reconciliation because the model cannot identify
|
|
||||||
them independently. Otherwise the helper converts returned descriptors to its
|
|
||||||
internal candidate keys before applying all existing unknown-member,
|
|
||||||
ineligible-member, duplicate-member, canonical-membership, overlap, retry, and
|
|
||||||
fallback rules.
|
|
||||||
|
|
||||||
## Stage 1: Convert NPC Occurrences To Name-Based Resolution
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Remove durable NPC IDs from the NPC-occurrence prompt and private response,
|
|
||||||
then resolve the model's contextual name deterministically without weakening
|
|
||||||
checkpoint identity or downstream validation.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Inspect:
|
|
||||||
- `assets/dnd/npc-occurrences/`;
|
|
||||||
- `internal/modules/dnd/extract/npcoccurrences/`;
|
|
||||||
- `internal/modules/dnd/npcs/registry/`;
|
|
||||||
- NPC-occurrence normalizer and validator checkpoint fingerprints; and
|
|
||||||
- their focused tests.
|
|
||||||
2. Change `dnd_npc_occurrences_llm.v1.json` so every occurrence requires only
|
|
||||||
`name`, `kind`, and `source_refs`, continues to reject unknown fields, and
|
|
||||||
no longer declares `npc_id`.
|
|
||||||
3. Revise the NPC-occurrence instructions to require a supplied canonical NPC
|
|
||||||
name and current-chunk evidence, with no instruction to copy or invent an
|
|
||||||
ID. Continue using the existing shared names-only NPC registry fragment and
|
|
||||||
preserve manifest order/cache controls.
|
|
||||||
4. Remove `NPCID` from the private `occurrenceResponse`. After canonicalizing
|
|
||||||
response evidence, resolve every response name with the existing
|
|
||||||
`npcregistry.Registry.Lookup` comparison-key lookup. On the first unknown
|
|
||||||
or non-unique selection, return an extractor-scoped mapping error and no
|
|
||||||
value. For a match, construct the durable occurrence with the registry
|
|
||||||
record's exact `ID` and canonical `Name`.
|
|
||||||
5. Change `mappingPolicy` to
|
|
||||||
`dnd.npc_occurrences.extract_mapping.v3`.
|
|
||||||
6. Stop passing `IdentityPromptInput()` to the LLM; use the existing
|
|
||||||
names-only `PromptInput()`.
|
|
||||||
7. Replace the misleading exported model-input API used only for identity
|
|
||||||
fingerprints: retain the unexported ordered `{id,name}` projection and its
|
|
||||||
digest, expose that value as `IdentityDigest() string`, remove
|
|
||||||
`IdentityPromptInput()`, and update NPC-occurrence extractor, normalizer,
|
|
||||||
invariant-validator, and registry-validator fingerprints to use
|
|
||||||
`IdentityDigest()`. The digest must still distinguish ID/name identity from
|
|
||||||
the names-only prompt projection.
|
|
||||||
8. Rewrite existing focused tests around observable behavior: rendered NPC
|
|
||||||
registry input is names-only; the private schema rejects `npc_id`; valid
|
|
||||||
names acquire the registry ID; comparison-equivalent names canonicalize;
|
|
||||||
unknown names fail the whole extraction; registry identity fingerprints
|
|
||||||
remain distinct and defensive; empty registries accept only empty model
|
|
||||||
results. Remove tests whose only purpose was requiring the model to return
|
|
||||||
exact ID/name pairs.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- No NPC-occurrence LLM input or private response contains a durable NPC ID.
|
|
||||||
- Durable NPC occurrences still contain the exact registry ID/name pair.
|
|
||||||
- Mapping failures remain extractor failures eligible for the configured
|
|
||||||
pipeline retry behavior.
|
|
||||||
- Deterministic consumers still fingerprint the ordered registry identity,
|
|
||||||
while spells, combat turns, enemy events, and NPC occurrences share the
|
|
||||||
names-only model projection.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/npcs/registry ./internal/modules/dnd/extract/npcoccurrences ./internal/modules/dnd/normalize/npcoccurrences ./internal/modules/dnd/validate/npcoccurrences/...
|
|
||||||
go test ./internal/modules/dnd/npcs/registry ./internal/modules/dnd/extract/npcoccurrences ./internal/modules/dnd/normalize/npcoccurrences ./internal/modules/dnd/validate/npcoccurrences/...
|
|
||||||
```
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
|
|
||||||
## Stage 2: Convert Item Occurrences To Name-Based Resolution
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Give item occurrences the same contextual-name/deterministic-ID boundary while
|
|
||||||
preserving item-specific nullable fields and kind rules.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Inspect `assets/dnd/item-occurrences/`, the item occurrence extractor, the
|
|
||||||
item registry, the item occurrence normalizer and registry validator, and
|
|
||||||
their focused tests.
|
|
||||||
2. Change `dnd_item_occurrences_llm.v1.json` to remove `item_id` from required
|
|
||||||
fields and properties. Preserve required `name`, `kind`, `quantity`, `from`,
|
|
||||||
`to`, and `source_refs`, all current enums/nullability, and strict unknown
|
|
||||||
field rejection.
|
|
||||||
3. Rewrite the item registry fragment and module instructions to require the
|
|
||||||
supplied canonical name and current-chunk evidence without mentioning an
|
|
||||||
ID. Preserve prompt order and cache controls.
|
|
||||||
4. Change the item registry's model projection from ordered `{id,name}` pairs
|
|
||||||
to ordered names-only objects, add a comparison-key index, and expose a
|
|
||||||
defensive `Lookup(name) (dnd.Item, bool)` analogous to the NPC registry.
|
|
||||||
Retain exact `LookupID` for durable normalizers and validators. Because item
|
|
||||||
IDs are derived from the item comparison identity, the names-only
|
|
||||||
`ProjectionDigest` remains sufficient for model input and existing
|
|
||||||
checkpoint consumers.
|
|
||||||
5. Remove `ItemID` from the private response. During response canonicalization,
|
|
||||||
resolve every contextual name, replace it with the registry record's
|
|
||||||
canonical name, and attach its durable ID when constructing the final
|
|
||||||
`dnd.ItemOccurrence`. Unknown selections fail the complete extraction; do
|
|
||||||
not alter evidence or nullable-field validation ownership.
|
|
||||||
6. Change `mappingPolicy` to
|
|
||||||
`dnd.item_occurrences.extract_mapping.v2`.
|
|
||||||
7. Update focused tests to cover names-only projection, defensive comparison
|
|
||||||
lookup, schema rejection of `item_id`, deterministic durable mapping,
|
|
||||||
unknown-name failure after an otherwise valid record, empty registry/result
|
|
||||||
behavior, and preservation of nullable/kind-specific fields.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Model-visible item registry and response content contain no item hash.
|
|
||||||
- Every accepted durable occurrence has the matched registry ID and canonical
|
|
||||||
name.
|
|
||||||
- Invalid selection remains all-or-nothing, and existing normalizer/validator
|
|
||||||
defense in depth remains unchanged.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/items/registry ./internal/modules/dnd/extract/itemoccurrences
|
|
||||||
go test ./internal/modules/dnd/items/registry ./internal/modules/dnd/extract/itemoccurrences ./internal/modules/dnd/normalize/itemoccurrences ./internal/modules/dnd/validate/itemoccurrences/...
|
|
||||||
```
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
|
|
||||||
## Stage 3: Add Contextual Location Grounding
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Replace the location registry's ID/name prompt projection with an immutable,
|
|
||||||
source-aware grounding object that can represent same-name locations safely.
|
|
||||||
Introduce the new path alongside the old occurrence input so this stage remains
|
|
||||||
buildable; Stage 4 performs the atomic extractor cutover and removes the old
|
|
||||||
path.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Inspect the location registry, location identity and source-reference
|
|
||||||
helpers, the generic source document index, occurrence checkpoint consumers,
|
|
||||||
and their focused tests.
|
|
||||||
2. In `internal/modules/dnd/locations/registry`, define the private-model
|
|
||||||
types needed by both grounding and the location occurrence extractor:
|
|
||||||
- a returned selector with exactly `name` and `registry_refs`;
|
|
||||||
- a registry projection entry with exactly `name`, `registry_refs`, and
|
|
||||||
`context`;
|
|
||||||
- a source-free range with exactly `start_unit_id` and `end_unit_id`; and
|
|
||||||
- a context unit with exactly `unit_id` and `text`.
|
|
||||||
All fields are required in their private JSON shapes, and constructors and
|
|
||||||
accessors must make defensive copies.
|
|
||||||
3. Add an operation-scoped immutable grounding type constructed from a resolved
|
|
||||||
registry and the current `*source.SourceDocument`. Its API must provide:
|
|
||||||
- a cloned `contracts.LLMInputMaterial` for the `location_registry` slot;
|
|
||||||
- deterministic resolution of a returned selector to one cloned
|
|
||||||
`dnd.Location`.
|
|
||||||
The prompt material's existing `Digest` field owns the digest of the exact
|
|
||||||
model projection; do not expose a second grounding-specific digest API.
|
|
||||||
4. Construct the projection in registry order. Group entries by the existing
|
|
||||||
location comparison key:
|
|
||||||
- every projection entry has exactly the required fields `name`,
|
|
||||||
`registry_refs`, and `context`;
|
|
||||||
- comparison-unique entries use empty `registry_refs` and `context` arrays;
|
|
||||||
- every same-name entry uses its complete canonical source ranges stripped
|
|
||||||
of `source_id` and includes ordered context units covered by those ranges;
|
|
||||||
- each context unit contains exactly required integer `unit_id` and string
|
|
||||||
`text` fields, and units are deduplicated in source order; and
|
|
||||||
- same-name ranges must have `SourceID == doc.ID` and pass
|
|
||||||
`source.DocumentIndex.ValidateRef`.
|
|
||||||
5. Fail grounding construction with a bounded, content-safe error if the
|
|
||||||
source is nil, a same-name range is invalid or belongs to another source,
|
|
||||||
a comparison key is empty, or two records produce the same selector. Do not
|
|
||||||
expose transcript text in the error.
|
|
||||||
6. Resolution uses the existing comparison key. A unique-name selector is
|
|
||||||
accepted only with empty `registry_refs`; a same-name selector is accepted
|
|
||||||
only on an exact canonical range match. Reject unknown names, a non-empty
|
|
||||||
range list for a unique name, an empty/partial/reordered range list for an
|
|
||||||
ambiguous name, or any selector not present in the grounding.
|
|
||||||
7. Separate deterministic identity fingerprinting from LLM material. Add
|
|
||||||
`IdentityDigest()` over the registry's ordered `{id,name}` identity
|
|
||||||
projection, and update the location normalizer and registry-validator
|
|
||||||
checkpoint consumers to use it. The new operation grounding carries the
|
|
||||||
model projection digest in its `LLMInputMaterial`. Retain the old ID-bearing
|
|
||||||
prompt accessor only as a documented transitional dependency of the
|
|
||||||
still-unchanged location occurrence extractor; do not add new callers.
|
|
||||||
8. Add focused tests for unique names, same-name context and selectors,
|
|
||||||
canonical range order, deterministic projection/digest, defensive copies,
|
|
||||||
exact selector resolution, nil/foreign/invalid references, selector
|
|
||||||
collisions, empty registries, and identity fingerprint stability. Do not
|
|
||||||
assert large rendered prompt strings; decode the JSON projection and assert
|
|
||||||
its semantic shape.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- The location package can build and resolve contextual selectors without
|
|
||||||
exposing `location_id`, `source_id`, digests, or replacement labels.
|
|
||||||
- Same-name locations remain distinct and receive meaningful bounded context.
|
|
||||||
- Deterministic checkpoint consumers retain an ID-sensitive fingerprint.
|
|
||||||
- Only the existing occurrence extractor remains wired to the legacy
|
|
||||||
ID-bearing prompt path until Stage 4; the repository compiles and tests pass.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/locations/registry ./internal/modules/dnd/normalize/locationoccurrences ./internal/modules/dnd/validate/locationoccurrences/...
|
|
||||||
go test ./internal/modules/dnd/locations/registry ./internal/modules/dnd/normalize/locationoccurrences ./internal/modules/dnd/validate/locationoccurrences/...
|
|
||||||
```
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
|
|
||||||
## Stage 4: Convert Location Occurrences To Contextual Resolution
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Wire the Stage 3 grounding object into location occurrence extraction and
|
|
||||||
remove durable location IDs from the prompt and private response.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Inspect `assets/dnd/location-occurrences/`, the location occurrence model,
|
|
||||||
schema loader, extractor, canonicalization, prompt tests, and Stage 3
|
|
||||||
grounding tests.
|
|
||||||
2. Change `dnd_location_occurrences_llm.v1.json` so each occurrence requires
|
|
||||||
exactly `name`, `registry_refs`, `kind`, and `source_refs`; remove
|
|
||||||
`location_id`. Keep all four occurrence kinds. Make `registry_refs` a
|
|
||||||
required array, including an empty array, of strict required positive
|
|
||||||
integer ranges. Keep occurrence `source_refs` separate and unchanged.
|
|
||||||
3. Rewrite `location-registry.md` and module instructions to explain the two
|
|
||||||
selector cases, require exact supplied contextual selectors, prohibit
|
|
||||||
invented locations, and state that registry ranges/context are identity
|
|
||||||
grounding rather than occurrence evidence. Preserve manifest order and
|
|
||||||
cache controls.
|
|
||||||
4. Change the private response type to `Name`, `RegistryRefs`, `Kind`, and
|
|
||||||
`SourceRefs`. Do not reuse `source.SourceRef` for the source-free registry
|
|
||||||
range type.
|
|
||||||
5. In `Extract`, construct operation grounding from the resolved registry and
|
|
||||||
`req.Source` before calling the LLM, put its projection in the
|
|
||||||
`location_registry` input, and resolve every returned selector after
|
|
||||||
completion. Attach the selected registry record's exact ID and canonical
|
|
||||||
name to the durable occurrence while retaining only the response's
|
|
||||||
current-source `source_refs` as evidence.
|
|
||||||
6. Fail the whole extraction on grounding-construction failure or the first
|
|
||||||
unknown, malformed, mismatched, or ambiguous selector. This replaces the
|
|
||||||
current behavior that can preserve unknown ID/name pairs for later
|
|
||||||
validators. Keep later normalizer and validator checks as defense in depth
|
|
||||||
for artifacts entering other boundaries.
|
|
||||||
7. Change `mappingPolicy` to
|
|
||||||
`dnd.location_occurrences.extract_mapping.v2`.
|
|
||||||
8. Remove the legacy ID-bearing registry `PromptInput` and its model-projection
|
|
||||||
digest once the extractor uses operation grounding. Keep the occurrence's
|
|
||||||
static module fingerprint based on `IdentityDigest()`, prompt/schema
|
|
||||||
fingerprints, and mapping policy. The operation-scoped projection is already
|
|
||||||
covered by source/chunk identity and configured or generated reference
|
|
||||||
dependencies, while its `LLMInputMaterial.Digest` identifies the exact model
|
|
||||||
input; do not add a second digest API or operation-aware static fingerprint.
|
|
||||||
9. Update focused schema, prompt, extractor, canonicalization, checkpoint, and
|
|
||||||
generated-reference tests. Cover unique-name empty selectors, successful
|
|
||||||
same-name selection, failure for an unsupported ambiguous mention,
|
|
||||||
partial/reordered ranges, no registry-to-occurrence evidence leakage,
|
|
||||||
all-or-nothing failure, empty registry/result behavior, and unchanged
|
|
||||||
durable ordering/deduplication.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- The location prompt and private response contain no durable location ID.
|
|
||||||
- Unique and same-name records resolve according to the final selector
|
|
||||||
contract.
|
|
||||||
- Accepted durable output is unchanged in shape and still contains an exact
|
|
||||||
location ID/name pair.
|
|
||||||
- Registry context cannot become durable occurrence evidence.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/extract/locationoccurrences ./internal/modules/dnd/locations/registry
|
|
||||||
go test ./internal/modules/dnd/locations/registry ./internal/modules/dnd/extract/locationoccurrences ./internal/modules/dnd/normalize/locationoccurrences ./internal/modules/dnd/validate/locationoccurrences/...
|
|
||||||
```
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
|
|
||||||
## Stage 5: Replace Reconciliation Keys With Contextual Descriptors
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Change the shared NPC/item/location registry-normalization proposal contract so
|
|
||||||
opaque candidate keys remain internal and the model sees and returns only
|
|
||||||
names plus evidence coordinates.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Inspect:
|
|
||||||
- `internal/modules/dnd/shared/entityreconcile/`;
|
|
||||||
- `assets/dnd/shared/prompts/common-dnd-entity-reconciliation.md`;
|
|
||||||
- `assets/dnd/entity-reconciliation/schemas/`;
|
|
||||||
- all three registry normalization manifests and prompt tests; and
|
|
||||||
- the NPC, item, and location registry normalizers and reconciliation tests.
|
|
||||||
2. Introduce one exported, defensively copied contextual selector type in
|
|
||||||
`entityreconcile` with JSON `name` and `source_refs`, plus a strict
|
|
||||||
source-free range type. Use it for candidate input views and for
|
|
||||||
`DuplicateGroup.Members` and `.Canonical`.
|
|
||||||
3. Keep deterministic candidate keys only inside `Materials`. During
|
|
||||||
`BuildContext`, validate and canonicalize candidate references as today,
|
|
||||||
serialize candidate views without `key`, derive a stable internal lookup
|
|
||||||
from canonical selector JSON to the corresponding internal candidate key,
|
|
||||||
and detect descriptor collisions before eligibility is established.
|
|
||||||
Colliding candidates must not appear in the prompt input or become
|
|
||||||
eligible; their records remain in deterministic normalization output.
|
|
||||||
4. Update `Materials.Assess` to resolve every returned selector through that
|
|
||||||
internal lookup before running the existing group assessment. Preserve
|
|
||||||
existing issue categories where their meaning still applies. Treat an
|
|
||||||
unknown or collided descriptor as an unknown/ineligible selection, discard
|
|
||||||
only the affected group, and retain existing overlap handling. `SafeGroup`
|
|
||||||
may continue returning internal candidate keys so the three domain
|
|
||||||
normalizers retain their position mapping; those keys are not model-facing.
|
|
||||||
5. Rewrite `dnd_entity_reconcile_llm.v1.json` so members and canonical are
|
|
||||||
strict selector objects. Require non-empty `name` structurally where the
|
|
||||||
current schemas do so, require `source_refs`, and make each range strict
|
|
||||||
with required positive integer endpoints. Preserve `duplicate_groups` and
|
|
||||||
the existing semantic assessment of minimum group size, membership,
|
|
||||||
duplicates, eligibility, and overlap rather than moving every semantic
|
|
||||||
failure into JSON Schema.
|
|
||||||
6. Rewrite the shared reconciliation fragment to tell the model to return
|
|
||||||
supplied contextual descriptors and never invent names or ranges. Remove
|
|
||||||
every instruction about opaque keys. Preserve all three manifests' message
|
|
||||||
ordering and cache controls.
|
|
||||||
7. Change registry normalization policy identifiers to:
|
|
||||||
- `dnd.npc_registry.normalize.v4`;
|
|
||||||
- `dnd.item_registry.normalize.v2`; and
|
|
||||||
- `dnd.location_registry.normalize.v2`.
|
|
||||||
8. Update shared and domain tests to cover candidate JSON without keys,
|
|
||||||
contextual proposal decoding, valid selector-to-internal-key mapping,
|
|
||||||
equal names with different evidence, descriptor collision exclusion,
|
|
||||||
unknown/partial/reordered descriptors, overlapping groups, canonical
|
|
||||||
membership, invalid structured-output fallback, currency safety, and
|
|
||||||
preservation of every non-applied deterministic candidate. Update prompt
|
|
||||||
asset fixtures to the new selector schema; do not snapshot prompt prose.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- No registry normalization prompt input or private response contains a
|
|
||||||
`candidate-*` key.
|
|
||||||
- Internal keys remain inaccessible to the model but may still support safe
|
|
||||||
deterministic position mapping.
|
|
||||||
- All existing normalizer safety, retry, fallback, warning, currency, and
|
|
||||||
same-name-location policies remain intact.
|
|
||||||
- Identical contextual descriptors cannot be arbitrarily reconciled.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/shared/entityreconcile ./internal/modules/dnd/normalize/npcregistry ./internal/modules/dnd/normalize/itemregistry ./internal/modules/dnd/normalize/locationregistry
|
|
||||||
go test ./internal/modules/dnd/shared/entityreconcile ./internal/modules/dnd/normalize/npcregistry ./internal/modules/dnd/normalize/itemregistry ./internal/modules/dnd/normalize/locationregistry
|
|
||||||
```
|
|
||||||
|
|
||||||
This is the largest stage, but it is one cohesive shared-contract migration
|
|
||||||
and is suitable for one gpt-5.6-terra prompt when implemented exactly within
|
|
||||||
the listed packages. Do not combine it with occurrence or documentation work.
|
|
||||||
|
|
||||||
## Stage 6: Record The Decision And Update Canonical Documentation
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Document the implemented policy in its durable architectural, internal, and
|
|
||||||
integration homes without duplicating volatile details or presenting roadmap
|
|
||||||
work as current behavior prematurely.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Re-read `docs/policy/documentation.md`, ADR-0003, ADR-0009, ADR-0011,
|
|
||||||
`docs/internal/dnd.md`, `docs/internal/llm.md`, and the six affected registry
|
|
||||||
and occurrence integration documents. Verify the code before describing it.
|
|
||||||
2. Add
|
|
||||||
`docs/adr/0012-resolve-opaque-entity-identifiers-deterministically.md` in
|
|
||||||
the repository's Nygard ADR format with status `Accepted` and the actual
|
|
||||||
implementation date. Record the model-semantic/deterministic-identity
|
|
||||||
boundary, source-coordinate allowance, request-local-label exception,
|
|
||||||
alternatives, ambiguity behavior, and consequences. Link ADR-0003 and
|
|
||||||
ADR-0009 rather than repeating their complete decisions.
|
|
||||||
3. Add a concise normative invariant under the LLM boundary in
|
|
||||||
`docs/policy/architecture.md`: callers use contextual model selections and
|
|
||||||
attach opaque application identities deterministically when possible. Link
|
|
||||||
ADR-0012 for rationale.
|
|
||||||
4. Update `docs/internal/dnd.md` to replace exact model-facing `{id,name}`
|
|
||||||
claims with the implemented NPC/item names-only and location contextual
|
|
||||||
selector behavior. Document reconciliation descriptors, internal-only keys,
|
|
||||||
all-or-nothing occurrence mapping failures, normalization fallback, and the
|
|
||||||
separation between registry and occurrence evidence. Do not duplicate the
|
|
||||||
private JSON schemas.
|
|
||||||
5. Add only a short ownership clarification to `docs/internal/llm.md`: the
|
|
||||||
calling module resolves contextual selections; PromptKit and its adapter do
|
|
||||||
not own entity identity.
|
|
||||||
6. Update these durable integration contracts while preserving their public
|
|
||||||
ID-bearing wire examples and schema statements:
|
|
||||||
- `docs/integrations/dnd-npc-registry-artifacts.md`;
|
|
||||||
- `docs/integrations/dnd-npc-occurrence-artifacts.md`;
|
|
||||||
- `docs/integrations/dnd-item-registry-artifacts.md`;
|
|
||||||
- `docs/integrations/dnd-item-occurrence-artifacts.md`;
|
|
||||||
- `docs/integrations/dnd-location-registry-artifacts.md`; and
|
|
||||||
- `docs/integrations/dnd-location-occurrence-artifacts.md`.
|
|
||||||
Remove claims that LLM consumers receive `{id,name}` or that raw model
|
|
||||||
output supplies an ID. State that Notarius maps contextual output into the
|
|
||||||
unchanged exact durable pair.
|
|
||||||
7. Revise the generic LLM-assisted deduplication entry in
|
|
||||||
`docs/roadmap/future.md`: stable unique IDs remain internal deterministic
|
|
||||||
state, while a future model proposal uses contextual descriptors or a
|
|
||||||
specifically justified request-local short label.
|
|
||||||
8. Do not change README, CLI, configuration, operations, examples, or public
|
|
||||||
schema files; this feature has no user-selectable surface or public wire
|
|
||||||
change.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- ADR-0012 owns rationale; architecture owns the normative boundary; internal
|
|
||||||
docs own mechanics; integration docs own unchanged durable contracts; and
|
|
||||||
the future roadmap no longer proposes durable IDs as the default model
|
|
||||||
selector.
|
|
||||||
- No current-behavior document claims that a model copies hash-based entity
|
|
||||||
IDs or opaque reconciliation keys.
|
|
||||||
- Documentation does not duplicate private schemas or implementation history.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git diff --check
|
|
||||||
rg -n '\{id,name\}|ID/name grounding|Candidate keys are opaque|candidate-[0-9]' docs assets/dnd
|
|
||||||
```
|
|
||||||
|
|
||||||
Review every search result semantically; durable wire-contract ID/name
|
|
||||||
requirements and internal test fixtures are not automatically errors.
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
|
|
||||||
## Stage 7: Integration Audit And Final Verification
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Verify the assembled D&D family, remove obsolete identity-copy paths, and
|
|
||||||
finish with a clean, policy-compliant implementation.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Audit every maintained D&D prompt manifest, selected fragment, private
|
|
||||||
schema, and constructed prompt projection. Confirm that no model is asked to
|
|
||||||
reproduce `npc:sha256:...`, `item:sha256:...`,
|
|
||||||
`location:sha256:...`, `candidate-*`, a UUID, a digest, or another opaque
|
|
||||||
entity handle. Do not confuse runtime metadata or durable output contracts
|
|
||||||
with model-visible material.
|
|
||||||
2. Trace all former APIs and fields, including `IdentityPromptInput`,
|
|
||||||
ID-bearing item/location prompt projections, private `NPCID`/`ItemID`/
|
|
||||||
`LocationID` response fields, and model-visible candidate keys. Remove dead
|
|
||||||
code, obsolete comments, stale test names, and unused assets. Retain
|
|
||||||
identity-only digests and exact durable lookup APIs used by deterministic
|
|
||||||
consumers.
|
|
||||||
3. Review prompt fingerprint registration and checkpoint fingerprints. Confirm
|
|
||||||
that each affected prompt/schema/policy/projection change invalidates the
|
|
||||||
relevant operation and that unrelated D&D lanes retain their existing
|
|
||||||
fingerprints.
|
|
||||||
4. Run representative production registration and multi-step pipeline tests
|
|
||||||
using existing fakes. Update only tests whose stable behavior changed.
|
|
||||||
Confirm generated NPC/item/location registry handoffs still prepare and
|
|
||||||
that final durable occurrences encode and validate under their existing
|
|
||||||
`v1` codecs.
|
|
||||||
5. Run formatting, focused suites, full tests, vet, build, and documentation
|
|
||||||
whitespace checks. Fix only failures caused by this feature. Report any
|
|
||||||
unrelated pre-existing failure without broadening scope.
|
|
||||||
6. Review the feature roadmap acceptance criteria one by one. Do not delete
|
|
||||||
`contextual-entity-grounding.md` or this implementation plan in this stage;
|
|
||||||
roadmap retirement is a separate maintainer action after review.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- All feature-roadmap acceptance criteria are met.
|
|
||||||
- The repository contains no obsolete model-facing opaque-identity path.
|
|
||||||
- Public artifacts and generated handoffs remain compatible.
|
|
||||||
- Tests are focused on behavior rather than prose or implementation shape.
|
|
||||||
- The worktree contains only intentional feature and documentation changes.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go fmt ./internal/modules/dnd/...
|
|
||||||
go test ./internal/modules/dnd/...
|
|
||||||
go test ./internal/modules/integration/...
|
|
||||||
go test ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./cmd/notarius
|
|
||||||
git diff --check
|
|
||||||
git status --short
|
|
||||||
```
|
|
||||||
|
|
||||||
This stage is suitable for one gpt-5.6-terra prompt.
|
|
||||||
7
go.mod
7
go.mod
@@ -3,9 +3,14 @@ module gitea.maximumdirect.net/eric/notarius
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0
|
gitea.maximumdirect.net/eric/promptkit v0.9.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.40.0
|
require golang.org/x/text v0.40.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -1,5 +1,9 @@
|
|||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
|
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
36
internal/buildinfo/buildinfo.go
Normal file
36
internal/buildinfo/buildinfo.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
// Package buildinfo resolves the product version embedded in a Notarius build.
|
||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"runtime/debug"
|
||||||
|
)
|
||||||
|
|
||||||
|
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
|
||||||
|
|
||||||
|
// Override is set at link time for controlled builds.
|
||||||
|
var Override string
|
||||||
|
|
||||||
|
// Version returns the release version embedded in the build, or development
|
||||||
|
// when the build does not carry a stable release tag.
|
||||||
|
func Version() (string, error) {
|
||||||
|
buildVersion := ""
|
||||||
|
if info, ok := debug.ReadBuildInfo(); ok {
|
||||||
|
buildVersion = info.Main.Version
|
||||||
|
}
|
||||||
|
return resolve(Override, buildVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolve(override, buildVersion string) (string, error) {
|
||||||
|
if override != "" {
|
||||||
|
if !stableVersion.MatchString(override) {
|
||||||
|
return "", fmt.Errorf("build version override is not a stable release tag")
|
||||||
|
}
|
||||||
|
return override, nil
|
||||||
|
}
|
||||||
|
if stableVersion.MatchString(buildVersion) {
|
||||||
|
return buildVersion, nil
|
||||||
|
}
|
||||||
|
return "development", nil
|
||||||
|
}
|
||||||
45
internal/buildinfo/buildinfo_test.go
Normal file
45
internal/buildinfo/buildinfo_test.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestResolve(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
buildVersion string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
|
||||||
|
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
|
||||||
|
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
|
||||||
|
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
|
||||||
|
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
|
||||||
|
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
|
||||||
|
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
|
||||||
|
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
|
||||||
|
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
|
||||||
|
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
|
||||||
|
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
|
||||||
|
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
|
||||||
|
{name: "development build", buildVersion: "(devel)", want: "development"},
|
||||||
|
{name: "missing build information", want: "development"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := resolve(tt.override, tt.buildVersion)
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("resolve() error = nil, want error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("resolve() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
81
internal/cli/assembled_enemy_event_codec_contract_test.go
Normal file
81
internal/cli/assembled_enemy_event_codec_contract_test.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
const invalidEnemyEventExtractorKey = "test/dnd/invalid-enemy-events"
|
||||||
|
|
||||||
|
func TestAssembledEnemyEventLaneRejectsInvalidFinalArtifactDespiteValidatorOverrides(t *testing.T) {
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
if err := pipeline.RegisterExtractor[dnd.EnemyEventList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||||
|
Key: invalidEnemyEventExtractorKey,
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
Requires: []string{"chunks", "source.transcript"},
|
||||||
|
Provides: []string{"dnd.enemy_events"},
|
||||||
|
ArtifactKind: dnd.EnemyEventListKind,
|
||||||
|
}, func() (contracts.Extractor[dnd.EnemyEventList], error) {
|
||||||
|
return invalidEnemyEventExtractor{}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register extractor: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
accept := pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")}}
|
||||||
|
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||||
|
ID: "assembled-invalid-enemy-events",
|
||||||
|
Input: pipeline.Binding("seriatim"),
|
||||||
|
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"enemy-events": {
|
||||||
|
Extract: pipeline.ModuleBinding{Module: invalidEnemyEventExtractorKey, Validators: accept},
|
||||||
|
Normalize: pipeline.ModuleBinding{Module: pipeline.DefaultNormalizeModule, Validators: accept},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Output: pipeline.Binding("json"),
|
||||||
|
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v", err)
|
||||||
|
}
|
||||||
|
_, err = pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||||
|
Prepared: prepared,
|
||||||
|
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||||
|
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "serialize accepted extract output") || !strings.Contains(err.Error(), "must not exceed") {
|
||||||
|
t.Fatalf("Run() error = %v, want final durable range rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type invalidEnemyEventExtractor struct{}
|
||||||
|
|
||||||
|
func (invalidEnemyEventExtractor) Key() string { return invalidEnemyEventExtractorKey }
|
||||||
|
|
||||||
|
func (invalidEnemyEventExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
|
func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, err
|
||||||
|
}
|
||||||
|
if req.Source == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, errors.New("assembled extractor requires source")
|
||||||
|
}
|
||||||
|
return contracts.TypedExtractionResult[dnd.EnemyEventList]{Value: dnd.EnemyEventList{Events: []dnd.EnemyEvent{{
|
||||||
|
Name: "Ashfang",
|
||||||
|
Kind: dnd.EnemyEventKindEngaged,
|
||||||
|
SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 2, EndUnitID: 1}},
|
||||||
|
}}}}, nil
|
||||||
|
}
|
||||||
@@ -21,6 +21,10 @@ import (
|
|||||||
|
|
||||||
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||||
|
|
||||||
|
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
|
||||||
|
|
||||||
|
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
|
||||||
|
|
||||||
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||||
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
||||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
@@ -67,28 +71,39 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
|||||||
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
|
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantWarningReasons := []string{
|
wantDiagnosticReasons := []string{
|
||||||
spellnormalize.ReasonCodeSpellNameCanonicalized,
|
spellnormalize.ReasonCodeSpellNameCanonicalized,
|
||||||
spellnormalize.ReasonCodeSourceReferencesNormalized,
|
spellnormalize.ReasonCodeSourceReferencesNormalized,
|
||||||
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
|
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
|
||||||
"spell_not_near_source",
|
"spell_not_near_source",
|
||||||
}
|
}
|
||||||
gotWarningReasons := make([]string, len(output.Warnings))
|
gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
|
||||||
for index, warning := range output.Warnings {
|
for index, group := range output.Diagnostics.Groups {
|
||||||
gotWarningReasons[index] = warning.ReasonCode
|
gotDiagnosticReasons[index] = group.ReasonCode
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
|
if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
|
||||||
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
|
||||||
}
|
}
|
||||||
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
|
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
|
||||||
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
|
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
|
||||||
}
|
}
|
||||||
|
|
||||||
warningsFile := decodeAssembledOutput[struct {
|
warningsFile := decodeAssembledOutput[struct {
|
||||||
Warnings []contracts.Warning `json:"warnings"`
|
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||||
}](t, output.OutputFiles, "warnings.json")
|
}](t, output.OutputFiles, "warnings.json")
|
||||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
if len(warningsFile.Groups) != 0 {
|
||||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
|
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
|
||||||
|
}
|
||||||
|
diagnosticsFile := decodeAssembledOutput[struct {
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
GroupCount int `json:"group_count"`
|
||||||
|
OccurrenceCount int `json:"occurrence_count"`
|
||||||
|
Truncated bool `json:"truncated"`
|
||||||
|
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
|
||||||
|
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||||
|
}](t, output.OutputFiles, "diagnostics.json")
|
||||||
|
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
|
||||||
|
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
|
||||||
}
|
}
|
||||||
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
||||||
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
||||||
@@ -101,6 +116,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
|
||||||
|
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
|
||||||
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||||
|
Prepared: prepared,
|
||||||
|
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||||
|
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
|
||||||
|
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
|
||||||
|
}
|
||||||
|
correction := extractor.correctionSnapshot()
|
||||||
|
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
|
||||||
|
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
||||||
var normalizeChain *pipeline.ResolvedValidatorChain
|
var normalizeChain *pipeline.ResolvedValidatorChain
|
||||||
@@ -127,15 +166,16 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
|||||||
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||||
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
||||||
}
|
}
|
||||||
for _, warning := range output.Warnings {
|
for _, group := range output.Diagnostics.Groups {
|
||||||
if warning.ReasonCode == "spell_not_near_source" {
|
if group.ReasonCode == "spell_not_near_source" {
|
||||||
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssembledSpellPipelineRejectsUnknownSpellWithoutPromotingAttemptWarning(t *testing.T) {
|
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
|
||||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
||||||
|
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
@@ -161,14 +201,12 @@ func TestAssembledSpellPipelineRejectsUnknownSpellWithoutPromotingAttemptWarning
|
|||||||
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
||||||
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
||||||
}
|
}
|
||||||
for _, warning := range output.Warnings {
|
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
|
||||||
if warning.ReasonCode == spellnormalize.ReasonCodeSpellNameUnresolved {
|
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
|
||||||
t.Fatalf("warnings = %#v, want rejected-attempt warning to remain non-durable", output.Warnings)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) {
|
func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
|
||||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
|
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
|
||||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -192,14 +230,20 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
|
|||||||
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
|
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
|
||||||
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
|
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
|
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
|
||||||
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
|
||||||
}
|
}
|
||||||
warningsFile := decodeAssembledOutput[struct {
|
warningsFile := decodeAssembledOutput[struct {
|
||||||
Warnings []contracts.Warning `json:"warnings"`
|
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||||
}](t, output.OutputFiles, "warnings.json")
|
}](t, output.OutputFiles, "warnings.json")
|
||||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
if len(warningsFile.Groups) != 0 {
|
||||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings)
|
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
|
||||||
|
}
|
||||||
|
diagnosticsFile := decodeAssembledOutput[struct {
|
||||||
|
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||||
|
}](t, output.OutputFiles, "diagnostics.json")
|
||||||
|
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
|
||||||
|
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,6 +252,47 @@ type assembledSpellPipelineOptions struct {
|
|||||||
unknownSpell bool
|
unknownSpell bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
|
||||||
|
t.Helper()
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
extractor := &assembledCorrectingSpellExtractor{}
|
||||||
|
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||||
|
Key: assembledCorrectingSpellExtractorKey,
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||||
|
Requires: []string{"chunks", "source.transcript"},
|
||||||
|
Provides: []string{"dnd.spell_casts"},
|
||||||
|
ArtifactKind: dnd.SpellListKind,
|
||||||
|
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||||
|
return extractor, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register correcting extractor: %v", err)
|
||||||
|
}
|
||||||
|
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
|
||||||
|
return assembledDirectSpellValidator{}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register direct spell validator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
|
||||||
|
extract.Retries = 1
|
||||||
|
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
|
||||||
|
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||||
|
ID: "assembled-dnd-correcting-spells",
|
||||||
|
Input: pipeline.Binding("seriatim"),
|
||||||
|
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
|
||||||
|
},
|
||||||
|
Output: pipeline.Binding("json"),
|
||||||
|
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return components.registries, resolved, extractor
|
||||||
|
}
|
||||||
|
|
||||||
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
@@ -253,6 +338,69 @@ type assembledSpellExtractor struct {
|
|||||||
unknownSpell bool
|
unknownSpell bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type assembledCorrectingSpellExtractor struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
correction *contracts.SemanticCorrection
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
|
||||||
|
|
||||||
|
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
|
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||||
|
if req.Source == nil || req.Chunk == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
|
||||||
|
}
|
||||||
|
response := `{"spell":"accepted"}`
|
||||||
|
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||||
|
if req.Chunk.Index == 0 && req.Correction == nil {
|
||||||
|
response = `{"spell":"Mysterious Burst"}`
|
||||||
|
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||||
|
}
|
||||||
|
if req.Chunk.Index == 0 && req.Correction != nil {
|
||||||
|
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
e.correction = correction
|
||||||
|
e.mu.Unlock()
|
||||||
|
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||||
|
}
|
||||||
|
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||||
|
}
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
correction, err := contracts.CloneSemanticCorrection(e.correction)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return correction
|
||||||
|
}
|
||||||
|
|
||||||
|
type assembledDirectSpellValidator struct{}
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
||||||
|
for _, cast := range req.Value.SpellCasts {
|
||||||
|
if cast.Spell == "Mysterious Burst" {
|
||||||
|
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
||||||
|
|
||||||
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
@@ -298,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
|
|||||||
return append([]int(nil), e.chunkIndexes...)
|
return append([]int(nil), e.chunkIndexes...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
|
||||||
|
count := 0
|
||||||
|
for _, group := range groups {
|
||||||
|
count += group.OccurrenceCount
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||||
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||||
|
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCommandVersionOutput(t *testing.T) {
|
||||||
|
previous := buildinfo.Override
|
||||||
|
t.Cleanup(func() { buildinfo.Override = previous })
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
wantCode int
|
||||||
|
wantStdout string
|
||||||
|
wantStderr string
|
||||||
|
}{
|
||||||
|
{name: "development", wantStdout: "notarius development\n"},
|
||||||
|
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
|
||||||
|
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buildinfo.Override = tt.override
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
|
||||||
|
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||||
environment := writeCommandConfig(t, "environment", "beta")
|
environment := writeCommandConfig(t, "environment", "beta")
|
||||||
|
|||||||
@@ -189,10 +189,18 @@ func TestMaintainedCompleteExamplePublishesRegistryBackedEntityOccurrences(t *te
|
|||||||
}
|
}
|
||||||
|
|
||||||
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
||||||
for _, laneID := range []string{"enemy-events", "npc-registry", "npc-occurrences", "item-registry", "item-occurrences", "location-registry", "location-occurrences"} {
|
if len(evidence) == 0 {
|
||||||
if !containsString(evidence.SelectedLanes, laneID) || !evidenceHasLane(evidence, laneID) {
|
t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
|
||||||
t.Fatalf("evidence context = %#v, want direct %s evidence", evidence, laneID)
|
|
||||||
}
|
}
|
||||||
|
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
|
||||||
|
for _, unit := range evidence {
|
||||||
|
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||||
|
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
|
||||||
|
}
|
||||||
|
if _, exists := seenEvidenceUnits[unit.ID]; exists {
|
||||||
|
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
|
||||||
|
}
|
||||||
|
seenEvidenceUnits[unit.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
requests := client.requestsFor(enemyevents.PromptID)
|
requests := client.requestsFor(enemyevents.PromptID)
|
||||||
@@ -330,7 +338,7 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
|
|||||||
if len(registry.Items) != 1 || registry.Items[0].Name != "Moonblade" {
|
if len(registry.Items) != 1 || registry.Items[0].Name != "Moonblade" {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated item registry has %d items, want 1", len(registry.Items))
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated item registry has %d items, want 1", len(registry.Items))
|
||||||
}
|
}
|
||||||
content = []byte(`{"occurrences":[{"name":"Moonblade","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_segment":5,"end_segment":5}]}]}`)
|
content = []byte(`{"occurrences":[{"name":"Moonblade","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":5,"end_unit_id":5}]}]}`)
|
||||||
}
|
}
|
||||||
case combat.PromptID:
|
case combat.PromptID:
|
||||||
content = []byte(`{"combat_turns":[{"actor":"Kesh","turn_kind":"turn","source_refs":[{"start_unit_id":8,"end_unit_id":8}]}]}`)
|
content = []byte(`{"combat_turns":[{"actor":"Kesh","turn_kind":"turn","source_refs":[{"start_unit_id":8,"end_unit_id":8}]}]}`)
|
||||||
@@ -388,8 +396,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
|
|||||||
if err := json.Unmarshal(content, output); err != nil {
|
if err := json.Unmarshal(content, output); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||||
}
|
}
|
||||||
|
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||||
|
}
|
||||||
client.mu.Lock()
|
client.mu.Lock()
|
||||||
client.requests = append(client.requests, request)
|
client.requests = append(client.requests, snapshot)
|
||||||
client.mu.Unlock()
|
client.mu.Unlock()
|
||||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
|
||||||
}
|
}
|
||||||
@@ -400,7 +412,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
|
|||||||
var requests []contracts.StructuredCompletionRequest
|
var requests []contracts.StructuredCompletionRequest
|
||||||
for _, request := range client.requests {
|
for _, request := range client.requests {
|
||||||
if request.PromptID == promptID {
|
if request.PromptID == promptID {
|
||||||
requests = append(requests, request)
|
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
requests = append(requests, snapshot)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return requests
|
return requests
|
||||||
@@ -415,17 +431,6 @@ func containsString(values []string, want string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func evidenceHasLane(value evidencecontext.Document, laneID string) bool {
|
|
||||||
for _, context := range value.Contexts {
|
|
||||||
for _, reference := range context.EvidenceRefs {
|
|
||||||
if reference.LaneID == laneID {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
||||||
for _, binding := range bindings {
|
for _, binding := range bindings {
|
||||||
if binding.SlotName == slotName && binding.Artifact != nil {
|
if binding.SlotName == slotName && binding.Artifact != nil {
|
||||||
|
|||||||
@@ -91,8 +91,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(durable, want) {
|
if !reflect.DeepEqual(durable, want) {
|
||||||
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
|
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 0 {
|
if len(output.Diagnostics.Groups) != 0 {
|
||||||
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -239,10 +239,10 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|||||||
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
||||||
}
|
}
|
||||||
warnings := readProductionJSON[struct {
|
warnings := readProductionJSON[struct {
|
||||||
Warnings []json.RawMessage `json:"warnings"`
|
Groups []json.RawMessage `json:"groups"`
|
||||||
}](t, filepath.Join(runRoot, "warnings.json"))
|
}](t, filepath.Join(runRoot, "warnings.json"))
|
||||||
if len(warnings.Warnings) != 0 {
|
if len(warnings.Groups) != 0 {
|
||||||
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
|
t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,12 +19,15 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||||
@@ -311,6 +314,81 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
|||||||
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||||
t.Fatalf("prepare production scene and spell modules: %v", err)
|
t.Fatalf("prepare production scene and spell modules: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
schemaFS, err := components.assets.SchemaFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production schema assets: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
||||||
|
t.Fatalf("generic reconciliation schema asset: %v", err)
|
||||||
|
}
|
||||||
|
options, err := components.assets.PromptKitOptions()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production PromptKit options: %v", err)
|
||||||
|
}
|
||||||
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
||||||
|
})))
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production prompt engine: %v", err)
|
||||||
|
}
|
||||||
|
promptFS, err := components.assets.PromptFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production prompt assets: %v", err)
|
||||||
|
}
|
||||||
|
type manifest struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Inputs []struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
} `yaml:"inputs"`
|
||||||
|
Messages []struct {
|
||||||
|
Role string `yaml:"role"`
|
||||||
|
} `yaml:"messages"`
|
||||||
|
}
|
||||||
|
preparedPrompts := 0
|
||||||
|
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(promptFS, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var prompt manifest
|
||||||
|
if err := yaml.Unmarshal(data, &prompt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, message := range prompt.Messages {
|
||||||
|
if message.Role != promptkit.RoleSystem && message.Role != promptkit.RoleUser {
|
||||||
|
return fmt.Errorf("production prompt %q uses role %q, want system or user", prompt.ID, message.Role)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
|
||||||
|
for _, input := range prompt.Inputs {
|
||||||
|
inputs[input.Name] = promptkit.Inline(`{}`)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
|
||||||
|
}
|
||||||
|
if prepared.OutputContract.RepairAttempts != 1 {
|
||||||
|
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
|
||||||
|
}
|
||||||
|
preparedPrompts++
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if preparedPrompts == 0 {
|
||||||
|
t.Fatal("prepared no production prompts")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
||||||
@@ -774,10 +852,10 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
|
|||||||
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
|
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
|
||||||
}
|
}
|
||||||
warnings := readProductionJSON[struct {
|
warnings := readProductionJSON[struct {
|
||||||
Warnings []contracts.Warning `json:"warnings"`
|
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||||
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
|
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
|
||||||
if len(warnings.Warnings) != 0 {
|
if len(warnings.Groups) != 0 {
|
||||||
t.Fatalf("warnings = %#v, want none", warnings.Warnings)
|
t.Fatalf("warnings = %#v, want none", warnings.Groups)
|
||||||
}
|
}
|
||||||
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
|
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
|
||||||
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
|
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
|
||||||
@@ -1021,8 +1099,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
|
|||||||
if err := json.Unmarshal(content, out); err != nil {
|
if err := json.Unmarshal(content, out); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||||
}
|
}
|
||||||
|
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||||
|
}
|
||||||
client.mu.Lock()
|
client.mu.Lock()
|
||||||
client.requests = append(client.requests, req)
|
client.requests = append(client.requests, snapshot)
|
||||||
client.mu.Unlock()
|
client.mu.Unlock()
|
||||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||||
}
|
}
|
||||||
@@ -1033,7 +1115,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
|
|||||||
var requests []contracts.StructuredCompletionRequest
|
var requests []contracts.StructuredCompletionRequest
|
||||||
for _, req := range client.requests {
|
for _, req := range client.requests {
|
||||||
if req.PromptID == promptID {
|
if req.PromptID == promptID {
|
||||||
requests = append(requests, req)
|
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
requests = append(requests, snapshot)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return requests
|
return requests
|
||||||
|
|||||||
@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "malformed profile",
|
name: "malformed profile",
|
||||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
|
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||||
profileID: "malformed-profile",
|
profileID: "malformed-profile",
|
||||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||||
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
|
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid profile source",
|
name: "invalid profile source",
|
||||||
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
|||||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||||
|
for _, profiles := range []string{
|
||||||
|
"id: child\nbase_profile: missing\n",
|
||||||
|
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||||
|
} {
|
||||||
|
t.Run("invalid inheritance", func(t *testing.T) {
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profileID := "child"
|
||||||
|
if strings.Contains(profiles, "id: first") {
|
||||||
|
profileID = "first"
|
||||||
|
}
|
||||||
|
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
|
||||||
|
t.Fatalf("profile preflight error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -400,6 +400,9 @@ func (referenceContractCodecA) Encode(stateTestArtifact) ([]byte, error) {
|
|||||||
func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) {
|
func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) {
|
||||||
return stateTestArtifact{Value: "ok"}, nil
|
return stateTestArtifact{Value: "ok"}, nil
|
||||||
}
|
}
|
||||||
|
func (codec referenceContractCodecA) DecodeCandidate(content []byte) (stateTestArtifact, error) {
|
||||||
|
return codec.Decode(content)
|
||||||
|
}
|
||||||
|
|
||||||
func (referenceContractCodecB) Kind() contracts.ArtifactKind { return referenceContractKindBeta }
|
func (referenceContractCodecB) Kind() contracts.ArtifactKind { return referenceContractKindBeta }
|
||||||
func (referenceContractCodecB) Schema() contracts.ArtifactSchema {
|
func (referenceContractCodecB) Schema() contracts.ArtifactSchema {
|
||||||
@@ -415,6 +418,9 @@ func (referenceContractCodecB) Encode(stateTestArtifact) ([]byte, error) {
|
|||||||
func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
|
func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
|
||||||
return stateTestArtifact{Value: "ok"}, nil
|
return stateTestArtifact{Value: "ok"}, nil
|
||||||
}
|
}
|
||||||
|
func (codec referenceContractCodecB) DecodeCandidate(content []byte) (stateTestArtifact, error) {
|
||||||
|
return codec.Decode(content)
|
||||||
|
}
|
||||||
|
|
||||||
func referenceContractLane(t *testing.T, resolved pipeline.ResolvedPipeline, id string) pipeline.ResolvedArtifactLane {
|
func referenceContractLane(t *testing.T, resolved pipeline.ResolvedPipeline, id string) pipeline.ResolvedArtifactLane {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
@@ -30,6 +32,7 @@ import (
|
|||||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
@@ -61,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
|
if len(args) > 0 && args[0] == "--version" {
|
||||||
|
if len(args) != 1 {
|
||||||
|
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
version, err := buildinfo.Version()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "notarius %s\n", version)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
opts, err = normalizeOptions(opts)
|
opts, err = normalizeOptions(opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -147,7 +164,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
machineOutput := fs.Bool("json", false, "write the successful run result as JSON")
|
machineOutput := fs.Bool("json", false, "write the successful run result as JSON")
|
||||||
debug := fs.Bool("debug", false, "write a debug bundle")
|
debug := fs.Bool("debug", false, "write a debug bundle")
|
||||||
debugDir := fs.String("debug-dir", "", "debug bundle directory")
|
debugDir := fs.String("debug-dir", "", "debug bundle directory")
|
||||||
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
llmProfile := singleValueFlag{name: "--llm-profile"}
|
||||||
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
|
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
|
||||||
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
|
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
|
||||||
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
|
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
|
||||||
@@ -157,6 +174,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
referenceFlags := stringListFlag{}
|
referenceFlags := stringListFlag{}
|
||||||
withoutReferenceFlags := stringListFlag{}
|
withoutReferenceFlags := stringListFlag{}
|
||||||
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
|
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
|
||||||
|
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
|
||||||
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
||||||
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||||
@@ -203,6 +221,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
|
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if llmProfile.set && strings.TrimSpace(llmProfile.value) == "" {
|
||||||
|
fmt.Fprintln(stderr, "notarius: --llm-profile must not be empty")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
if reasoningEffort.set && *clearReasoningEffort {
|
if reasoningEffort.set && *clearReasoningEffort {
|
||||||
fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
|
fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
|
||||||
return 2
|
return 2
|
||||||
@@ -338,7 +360,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
PipelineID: pipelineID,
|
PipelineID: pipelineID,
|
||||||
Only: only,
|
Only: only,
|
||||||
Catalog: catalog,
|
Catalog: catalog,
|
||||||
LLMProfileOverride: *llmProfile,
|
LLMProfileOverride: strings.TrimSpace(llmProfile.value),
|
||||||
ReferenceOverrides: referenceOverrides,
|
ReferenceOverrides: referenceOverrides,
|
||||||
ReferenceUnbinds: referenceUnbinds,
|
ReferenceUnbinds: referenceUnbinds,
|
||||||
})
|
})
|
||||||
@@ -353,7 +375,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
|
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
|
||||||
}
|
}
|
||||||
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
||||||
ConfigPath: loadedConfigPath,
|
ConfigPath: loadedConfigPath,
|
||||||
WorkingDir: workingDir,
|
WorkingDir: workingDir,
|
||||||
})
|
})
|
||||||
@@ -426,7 +448,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||||
}
|
}
|
||||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), effectiveSessionID, runtimeOverrides, *resume)
|
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||||
}
|
}
|
||||||
@@ -440,7 +462,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
StartedAt: startedAt,
|
StartedAt: startedAt,
|
||||||
LLMProfiles: llmProfiles,
|
LLMProfiles: llmProfiles,
|
||||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||||
Warnings: referenceWarnings,
|
Diagnostics: referenceDiagnostics,
|
||||||
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
|
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
|
||||||
ChunkPlans: chunkPlans,
|
ChunkPlans: chunkPlans,
|
||||||
Checkpoints: checkpointRecorder,
|
Checkpoints: checkpointRecorder,
|
||||||
@@ -449,7 +471,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
Debug: debugRecorder,
|
Debug: debugRecorder,
|
||||||
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
|
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
|
||||||
})
|
})
|
||||||
commandState.observeOutput(output)
|
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||||
|
if diagnosticErr == nil {
|
||||||
|
commandState.observeOutput(output, diagnosticProjection)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
|
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
|
||||||
if output.Manifest.PipelineID != "" {
|
if output.Manifest.PipelineID != "" {
|
||||||
@@ -457,15 +482,21 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
|
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if diagnosticErr != nil {
|
||||||
|
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||||
|
}
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
|
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
|
||||||
}
|
}
|
||||||
|
if diagnosticErr != nil {
|
||||||
|
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||||
|
}
|
||||||
|
|
||||||
if err := writePartialSummary(summary, output); err != nil {
|
if err := writePartialSummary(summary, output); err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
|
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
|
||||||
}
|
}
|
||||||
var encodedResult []byte
|
var encodedResult []byte
|
||||||
if *machineOutput {
|
if *machineOutput {
|
||||||
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
|
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||||
}
|
}
|
||||||
@@ -491,12 +522,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
|
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(output.Warnings) > 0 {
|
if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 {
|
||||||
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
|
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount)
|
||||||
|
if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok {
|
||||||
|
fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile))
|
||||||
|
}
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
}
|
}
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) {
|
||||||
|
for _, file := range files {
|
||||||
|
if file.Name == name {
|
||||||
|
return file.Name, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
|
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
|
||||||
if summary == nil {
|
if summary == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -516,7 +560,7 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := summary.WriteWarnings(output.Warnings); err != nil {
|
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return summary.WriteCheckpointEvents(output.CheckpointEvents)
|
return summary.WriteCheckpointEvents(output.CheckpointEvents)
|
||||||
@@ -752,17 +796,10 @@ func configSource(configPath string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error {
|
func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error {
|
||||||
type outputTarget struct {
|
|
||||||
path string
|
|
||||||
file contracts.OutputFile
|
|
||||||
}
|
|
||||||
targets := make([]outputTarget, 0, len(files))
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
targetPath, err := outputFilePath(runOutputDir, file.Name)
|
if _, err := outputFilePath(runOutputDir, file.Name); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
targets = append(targets, outputTarget{path: targetPath, file: file})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
outputParent := filepath.Dir(runOutputDir)
|
outputParent := filepath.Dir(runOutputDir)
|
||||||
@@ -775,12 +812,9 @@ func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error {
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("create output run directory %q: %w", runOutputDir, err)
|
return fmt.Errorf("create output run directory %q: %w", runOutputDir, err)
|
||||||
}
|
}
|
||||||
for _, target := range targets {
|
for _, file := range files {
|
||||||
if err := os.MkdirAll(filepath.Dir(target.path), 0o755); err != nil {
|
if err := fileio.WriteBytes(runOutputDir, file.Name, file.Bytes, 0o755, 0o644); err != nil {
|
||||||
return fmt.Errorf("create output directory %q: %w", filepath.Dir(target.path), err)
|
return fmt.Errorf("write output file %q: %w", file.Name, err)
|
||||||
}
|
|
||||||
if err := writeFileAtomic(target.path, target.file.Bytes, 0o644); err != nil {
|
|
||||||
return fmt.Errorf("write output file %q: %w", target.file.Name, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -823,38 +857,6 @@ func outputFilePath(runOutputDir, logicalName string) (string, error) {
|
|||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
tempPath := temp.Name()
|
|
||||||
removeTemp := true
|
|
||||||
defer func() {
|
|
||||||
if removeTemp {
|
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
if _, err := temp.Write(data); err != nil {
|
|
||||||
_ = temp.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := temp.Chmod(perm); err != nil {
|
|
||||||
_ = temp.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := temp.Close(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := os.Rename(tempPath, path); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
removeTemp = false
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func reorderRunArgs(args []string) []string {
|
func reorderRunArgs(args []string) []string {
|
||||||
var flags []string
|
var flags []string
|
||||||
var positionals []string
|
var positionals []string
|
||||||
|
|||||||
@@ -43,6 +43,12 @@ func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
|
|||||||
{name: "blank session ID", args: func(roots stateTestRoots) []string {
|
{name: "blank session ID", args: func(roots stateTestRoots) []string {
|
||||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
|
||||||
}},
|
}},
|
||||||
|
{name: "blank LLM profile", args: func(roots stateTestRoots) []string {
|
||||||
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", ""}
|
||||||
|
}},
|
||||||
|
{name: "whitespace LLM profile", args: func(roots stateTestRoots) []string {
|
||||||
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", " \t "}
|
||||||
|
}},
|
||||||
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
|
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
|
||||||
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
|
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
|
||||||
}},
|
}},
|
||||||
@@ -584,10 +590,11 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
|||||||
roots := newStateTestRoots(t)
|
roots := newStateTestRoots(t)
|
||||||
harness := newStateTestHarness()
|
harness := newStateTestHarness()
|
||||||
harness.includeWarnings = true
|
harness.includeWarnings = true
|
||||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
harness.includeWarningFile = true
|
||||||
|
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
||||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
|
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
}
|
}
|
||||||
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
||||||
@@ -595,11 +602,12 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
|||||||
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
||||||
t.Fatalf("durable output = %q, %v", output, err)
|
t.Fatalf("durable output = %q, %v", output, err)
|
||||||
}
|
}
|
||||||
|
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
|
||||||
bundle := onlyChildDir(t, roots.debug)
|
bundle := onlyChildDir(t, roots.debug)
|
||||||
var warnings []contracts.Warning
|
var diagnostics contracts.DiagnosticCollection
|
||||||
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
|
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
|
||||||
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
|
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
|
||||||
t.Fatalf("debug warnings = %#v", warnings)
|
t.Fatalf("debug diagnostics = %#v", diagnostics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,10 +38,36 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
|
|||||||
if err := writeOutputFiles(runPath, []contracts.OutputFile{{Name: "nested/result.json", Bytes: []byte("result")}}); err != nil {
|
if err := writeOutputFiles(runPath, []contracts.OutputFile{{Name: "nested/result.json", Bytes: []byte("result")}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
data, err := os.ReadFile(filepath.Join(runPath, "nested", "result.json"))
|
resultPath := filepath.Join(runPath, "nested", "result.json")
|
||||||
|
data, err := os.ReadFile(resultPath)
|
||||||
if err != nil || string(data) != "result" {
|
if err != nil || string(data) != "result" {
|
||||||
t.Fatalf("nested output = %q, %v", data, err)
|
t.Fatalf("nested output = %q, %v", data, err)
|
||||||
}
|
}
|
||||||
|
for path, want := range map[string]os.FileMode{runPath: 0o755, filepath.Join(runPath, "nested"): 0o755, resultPath: 0o644} {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mode := info.Mode().Perm()
|
||||||
|
if mode&^want != 0 {
|
||||||
|
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
|
||||||
|
}
|
||||||
|
if info.IsDir() && mode&0o700 != 0o700 {
|
||||||
|
t.Fatalf("%s mode = %#o, want owner access", path, mode)
|
||||||
|
}
|
||||||
|
if !info.IsDir() && mode != want {
|
||||||
|
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if strings.Contains(entry.Name(), ".tmp-") {
|
||||||
|
t.Fatalf("temporary file remains: %s", entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteOutputFilesRejectsUnsafeNamesBeforeAllocatingRunDirectory(t *testing.T) {
|
func TestWriteOutputFilesRejectsUnsafeNamesBeforeAllocatingRunDirectory(t *testing.T) {
|
||||||
@@ -75,7 +101,7 @@ func TestWriteOutputFilesRetainsNewPartialDirectoryAndPreservesSibling(t *testin
|
|||||||
{Name: "blocked", Bytes: []byte("partial output")},
|
{Name: "blocked", Bytes: []byte("partial output")},
|
||||||
{Name: "blocked/nested.json", Bytes: []byte("unreachable")},
|
{Name: "blocked/nested.json", Bytes: []byte("unreachable")},
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "create output directory") {
|
if err == nil || !strings.Contains(err.Error(), `write output file "blocked/nested.json"`) {
|
||||||
t.Fatalf("writeOutputFiles() error = %v, want later directory failure", err)
|
t.Fatalf("writeOutputFiles() error = %v, want later directory failure", err)
|
||||||
}
|
}
|
||||||
if got, err := os.ReadFile(filepath.Join(runPath, "blocked")); err != nil || string(got) != "partial output" {
|
if got, err := os.ReadFile(filepath.Join(runPath, "blocked")); err != nil || string(got) != "partial output" {
|
||||||
|
|||||||
@@ -7,10 +7,12 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
const runResultSchemaVersion = "notarius.run-result.v1"
|
const runResultSchemaVersion = "notarius.run-result.v2"
|
||||||
|
|
||||||
type runResult struct {
|
type runResult struct {
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
@@ -20,12 +22,25 @@ type runResult struct {
|
|||||||
IndexFile string `json:"index_file,omitempty"`
|
IndexFile string `json:"index_file,omitempty"`
|
||||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
NormalizedOutputCount int `json:"normalized_output_count"`
|
||||||
RejectedOutputCount int `json:"rejected_output_count"`
|
RejectedOutputCount int `json:"rejected_output_count"`
|
||||||
WarningCount int `json:"warning_count"`
|
WarningGroupCount int `json:"warning_group_count"`
|
||||||
|
WarningOccurrenceCount int `json:"warning_occurrence_count"`
|
||||||
|
DiagnosticGroupCount int `json:"diagnostic_group_count"`
|
||||||
|
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
|
||||||
|
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
|
||||||
ValidationStatus string `json:"validation_status"`
|
ValidationStatus string `json:"validation_status"`
|
||||||
|
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
||||||
|
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||||
|
if err != nil {
|
||||||
|
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
|
||||||
|
}
|
||||||
|
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
|
||||||
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
||||||
return runResult{}, fmt.Errorf("run result requires a run ID")
|
return runResult{}, fmt.Errorf("run result requires a run ID")
|
||||||
}
|
}
|
||||||
@@ -57,8 +72,13 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
|||||||
OutputDirectory: absOutputDirectory,
|
OutputDirectory: absOutputDirectory,
|
||||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
NormalizedOutputCount: len(output.NormalizeOutputs),
|
||||||
RejectedOutputCount: len(output.Rejected),
|
RejectedOutputCount: len(output.Rejected),
|
||||||
WarningCount: len(output.Warnings),
|
WarningGroupCount: len(diagnosticProjection.Warnings),
|
||||||
|
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
|
||||||
|
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
|
||||||
|
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
|
||||||
|
DiagnosticsTruncated: output.Diagnostics.Truncated,
|
||||||
ValidationStatus: output.Manifest.ValidationStatus,
|
ValidationStatus: output.Manifest.ValidationStatus,
|
||||||
|
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.TrimSpace(debugDirectory) != "" {
|
if strings.TrimSpace(debugDirectory) != "" {
|
||||||
@@ -85,6 +105,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
||||||
|
if len(summaries) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
||||||
|
for index, summary := range summaries {
|
||||||
|
cloned[index] = artifacts.CloneValidationSummary(summary)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
func encodeRunResult(result runResult) ([]byte, error) {
|
func encodeRunResult(result runResult) ([]byte, error) {
|
||||||
encoded, err := json.Marshal(result)
|
encoded, err := json.Marshal(result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
receipt := decodeRunResultDocument(t, stdout.String())
|
receipt := decodeRunResultDocument(t, stdout.String())
|
||||||
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
|
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
|
||||||
t.Fatalf("schema_version = %q", got)
|
t.Fatalf("schema_version = %q", got)
|
||||||
}
|
}
|
||||||
if got := receipt["run_id"]; got != productionRunID {
|
if got := receipt["run_id"]; got != productionRunID {
|
||||||
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
|||||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||||
t.Fatalf("rejected_output_count = %v", got)
|
t.Fatalf("rejected_output_count = %v", got)
|
||||||
}
|
}
|
||||||
if got := receipt["warning_count"]; got != float64(0) {
|
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||||
t.Fatalf("warning_count = %v", got)
|
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||||
}
|
}
|
||||||
if got := receipt["validation_status"]; got != "approved" {
|
if got := receipt["validation_status"]; got != "approved" {
|
||||||
t.Fatalf("validation_status = %q", got)
|
t.Fatalf("validation_status = %q", got)
|
||||||
@@ -63,19 +63,19 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
|||||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||||
roots := newStateTestRoots(t)
|
roots := newStateTestRoots(t)
|
||||||
harness := newStateTestHarness()
|
harness := newStateTestHarness()
|
||||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
code := RunWithOptions([]string{
|
code := RunWithOptions([]string{
|
||||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||||
"--chunk_cache", "bypass", "--debug", "--json",
|
"--chunk_cache", "bypass", "--debug", "--json",
|
||||||
}, &stdout, &stderr, harness.options())
|
}, &stdout, &stderr, harness.options())
|
||||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
|
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
receipt := decodeRunResultDocument(t, stdout.String())
|
receipt := decodeRunResultDocument(t, stdout.String())
|
||||||
if got := receipt["warning_count"]; got != float64(1) {
|
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||||
t.Fatalf("warning_count = %v", got)
|
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||||
}
|
}
|
||||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||||
@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
||||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,8 +52,11 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
|||||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
if got := decoded["rejected_output_count"]; got != float64(1) {
|
||||||
t.Fatalf("rejected_output_count = %v", got)
|
t.Fatalf("rejected_output_count = %v", got)
|
||||||
}
|
}
|
||||||
if got := decoded["warning_count"]; got != float64(1) {
|
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
|
||||||
t.Fatalf("warning_count = %v", got)
|
t.Fatalf("diagnostic counts = %#v", decoded)
|
||||||
|
}
|
||||||
|
if got := decoded["validation_summaries"]; got != nil {
|
||||||
|
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
||||||
}
|
}
|
||||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||||
t.Fatalf("output_directory = %q", got)
|
t.Fatalf("output_directory = %q", got)
|
||||||
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
||||||
|
output := testRunOutput()
|
||||||
|
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
||||||
|
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
||||||
|
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
||||||
|
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
||||||
|
}
|
||||||
|
encoded, err := encodeRunResult(result)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
||||||
|
t.Fatalf("encoded result = %s", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -159,7 +182,14 @@ func testRunOutput() pipeline.RunOutput {
|
|||||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
||||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
||||||
Rejected: []contracts.RejectedOutput{{}},
|
Rejected: []contracts.RejectedOutput{{}},
|
||||||
Warnings: []contracts.Warning{{}},
|
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
|
||||||
|
Disposition: contracts.DiagnosticDispositionWarning,
|
||||||
|
Category: contracts.DiagnosticCategoryFallback,
|
||||||
|
ReasonCode: "fallback",
|
||||||
|
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
||||||
|
OccurrenceCount: 1,
|
||||||
|
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
|
||||||
|
}}},
|
||||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -33,13 +34,17 @@ func (s *pipelineCommandState) setDebugPath(debugPath string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
|
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput, diagnostics contracts.DiagnosticProjection) {
|
||||||
if s == nil {
|
if s == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.report.OutputCount = len(output.NormalizeOutputs)
|
s.report.OutputCount = len(output.NormalizeOutputs)
|
||||||
s.report.RejectedCount = len(output.Rejected)
|
s.report.RejectedCount = len(output.Rejected)
|
||||||
s.report.WarningCount = len(output.Warnings)
|
s.report.WarningGroupCount = len(diagnostics.Warnings)
|
||||||
|
s.report.WarningOccurrenceCount = diagnostics.WarningOccurrenceCount
|
||||||
|
s.report.DiagnosticGroupCount = len(diagnostics.Diagnostics)
|
||||||
|
s.report.DiagnosticOccurrenceCount = diagnostics.DiagnosticOccurrenceCount
|
||||||
|
s.report.DiagnosticsTruncated = output.Diagnostics.Truncated
|
||||||
s.report.ValidationStatus = output.Manifest.ValidationStatus
|
s.report.ValidationStatus = output.Manifest.ValidationStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -238,7 +238,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
|
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -410,8 +410,8 @@ func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t
|
|||||||
t.Fatalf("spell requests = %d, want one", len(requests))
|
t.Fatalf("spell requests = %d, want one", len(requests))
|
||||||
}
|
}
|
||||||
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||||
if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
|
if !ok || !strings.Contains(string(catalogInput.Content), `"canonical_name":"Aegis of Emberfall"`) || !strings.Contains(string(catalogInput.Content), `"aliases":["Emberfall Aegis"]`) {
|
||||||
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
|
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name and recognition alias", catalogInput)
|
||||||
}
|
}
|
||||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
||||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
wantCalls int
|
wantCalls int
|
||||||
wantRejected bool
|
wantRejected bool
|
||||||
wantSpell string
|
wantSpell string
|
||||||
wantWarningCode string
|
wantAdvisoryCode string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "unknown spell remains rejected after exhaustion",
|
name: "unknown spell remains rejected after exhaustion",
|
||||||
@@ -44,7 +44,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
},
|
},
|
||||||
wantCalls: 2,
|
wantCalls: 2,
|
||||||
wantSpell: "Aegis of Emberfall",
|
wantSpell: "Aegis of Emberfall",
|
||||||
wantWarningCode: "spell_not_near_source",
|
wantAdvisoryCode: "spell_not_near_source",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
t.Fatalf("materialize production references: %v", err)
|
t.Fatalf("materialize production references: %v", err)
|
||||||
}
|
}
|
||||||
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
||||||
|
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||||
|
|
||||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||||
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 0 {
|
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
|
||||||
t.Fatalf("warnings = %#v, want no warnings from rejected attempts", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -107,8 +108,16 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode {
|
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
|
||||||
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings)
|
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
|
||||||
|
}
|
||||||
|
if len(output.Diagnostics.Groups) != 2 {
|
||||||
|
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range output.Diagnostics.Groups {
|
||||||
|
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
|
||||||
|
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -349,7 +349,7 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
|
|||||||
t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID)
|
t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID)
|
||||||
}
|
}
|
||||||
report := readStateTestRunReport(t, debugPath)
|
report := readStateTestRunReport(t, debugPath)
|
||||||
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
|
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "approved" {
|
||||||
t.Fatalf("success report = %#v", report)
|
t.Fatalf("success report = %#v", report)
|
||||||
}
|
}
|
||||||
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
|
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
|
||||||
@@ -431,7 +431,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
|
|||||||
bundlePath := onlyChildDir(t, roots.debug)
|
bundlePath := onlyChildDir(t, roots.debug)
|
||||||
runID := filepath.Base(bundlePath)
|
runID := filepath.Base(bundlePath)
|
||||||
report := readStateTestRunReport(t, bundlePath)
|
report := readStateTestRunReport(t, bundlePath)
|
||||||
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
|
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != tc.wantValidation {
|
||||||
t.Fatalf("failure report = %#v", report)
|
t.Fatalf("failure report = %#v", report)
|
||||||
}
|
}
|
||||||
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
||||||
@@ -445,7 +445,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
|
|||||||
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
||||||
roots := newStateTestRoots(t)
|
roots := newStateTestRoots(t)
|
||||||
harness := newStateTestHarness()
|
harness := newStateTestHarness()
|
||||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
|
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")}
|
||||||
harness.extractErr = errors.New("synthetic partial pipeline failure")
|
harness.extractErr = errors.New("synthetic partial pipeline failure")
|
||||||
|
|
||||||
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
|
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
|
||||||
@@ -454,7 +454,7 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
bundlePath := onlyChildDir(t, roots.debug)
|
bundlePath := onlyChildDir(t, roots.debug)
|
||||||
report := readStateTestRunReport(t, bundlePath)
|
report := readStateTestRunReport(t, bundlePath)
|
||||||
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
|
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningGroupCount != 1 || report.WarningOccurrenceCount != 1 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "failed" {
|
||||||
t.Fatalf("partial failure report = %#v", report)
|
t.Fatalf("partial failure report = %#v", report)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,10 +463,10 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
|||||||
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
|
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
|
||||||
t.Fatalf("partial manifest = %#v", manifest)
|
t.Fatalf("partial manifest = %#v", manifest)
|
||||||
}
|
}
|
||||||
var warnings []contracts.Warning
|
var diagnostics contracts.DiagnosticCollection
|
||||||
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
|
readStateTestSummaryJSON(t, bundlePath, "final-diagnostics.json", &diagnostics)
|
||||||
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
|
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" {
|
||||||
t.Fatalf("partial warnings = %#v", warnings)
|
t.Fatalf("partial diagnostics = %#v", diagnostics)
|
||||||
}
|
}
|
||||||
var events []pipeline.CheckpointEvent
|
var events []pipeline.CheckpointEvent
|
||||||
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
|
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
|
||||||
@@ -863,11 +863,12 @@ type stateTestHarness struct {
|
|||||||
chunkCalls, extractCalls int
|
chunkCalls, extractCalls int
|
||||||
runIDCalls uint64
|
runIDCalls uint64
|
||||||
extractErr error
|
extractErr error
|
||||||
chunkWarnings []contracts.Warning
|
chunkDiagnostics []contracts.ProducerDiagnostic
|
||||||
moduleProfiles []string
|
moduleProfiles []string
|
||||||
sessionIDs []string
|
sessionIDs []string
|
||||||
outputWarnings []contracts.Warning
|
outputDiagnostics contracts.DiagnosticCollection
|
||||||
includeWarnings bool
|
includeWarnings bool
|
||||||
|
includeWarningFile bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
||||||
@@ -892,7 +893,7 @@ func (h *stateTestHarness) options() Options {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||||
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
|
return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@@ -931,7 +932,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
|
|||||||
c.harness.mu.Lock()
|
c.harness.mu.Lock()
|
||||||
c.harness.chunkCalls++
|
c.harness.chunkCalls++
|
||||||
c.harness.mu.Unlock()
|
c.harness.mu.Unlock()
|
||||||
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
|
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
|
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
|
||||||
@@ -955,6 +956,9 @@ func (stateTestCodec) Encode(v stateTestArtifact) ([]byte, error) {
|
|||||||
func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) {
|
func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) {
|
||||||
return stateTestArtifact{Value: "ok"}, nil
|
return stateTestArtifact{Value: "ok"}, nil
|
||||||
}
|
}
|
||||||
|
func (codec stateTestCodec) DecodeCandidate(content []byte) (stateTestArtifact, error) {
|
||||||
|
return codec.Decode(content)
|
||||||
|
}
|
||||||
|
|
||||||
type stateTestExtractor struct{ harness *stateTestHarness }
|
type stateTestExtractor struct{ harness *stateTestHarness }
|
||||||
|
|
||||||
@@ -997,19 +1001,27 @@ func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNor
|
|||||||
|
|
||||||
type stateTestOutput struct {
|
type stateTestOutput struct {
|
||||||
harness *stateTestHarness
|
harness *stateTestHarness
|
||||||
includeWarnings bool
|
includeDiagnostics bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o stateTestOutput) Key() string { return "test/output" }
|
func (o stateTestOutput) Key() string { return "test/output" }
|
||||||
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||||
o.harness.mu.Lock()
|
o.harness.mu.Lock()
|
||||||
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
|
o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics)
|
||||||
o.harness.mu.Unlock()
|
o.harness.mu.Unlock()
|
||||||
data := []byte("{\"ok\":true}\n")
|
data := []byte("{\"ok\":true}\n")
|
||||||
if o.includeWarnings && len(req.Warnings) > 0 {
|
if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
|
||||||
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
|
data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
|
||||||
}
|
}
|
||||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
|
files := []contracts.OutputFile{{Name: "result.json", Bytes: data}}
|
||||||
|
if o.harness.includeWarningFile {
|
||||||
|
files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")})
|
||||||
|
}
|
||||||
|
return contracts.OutputResult{Files: files}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {
|
||||||
|
return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
type failingDebugRecorder struct{}
|
type failingDebugRecorder struct{}
|
||||||
|
|||||||
@@ -104,6 +104,33 @@ type RejectedOutputManifest struct {
|
|||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
AttemptCount int `json:"attempt_count,omitempty"`
|
AttemptCount int `json:"attempt_count,omitempty"`
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||||
|
Validation *ValidationSummary `json:"validation,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidationSummary is the bounded, durable outcome of validating one
|
||||||
|
// producer result. It deliberately contains identities and stable codes, not
|
||||||
|
// model responses, corrective guidance, validator diagnostics, or payloads.
|
||||||
|
type ValidationSummary struct {
|
||||||
|
Stage string `json:"stage,omitempty"`
|
||||||
|
StepID string `json:"step_id,omitempty"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key,omitempty"`
|
||||||
|
ChunkID string `json:"chunk_id,omitempty"`
|
||||||
|
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
RejectingValidators []string `json:"rejecting_validators,omitempty"`
|
||||||
|
ReasonCodes []string `json:"reason_codes,omitempty"`
|
||||||
|
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
|
||||||
|
ProducerAttemptCount int `json:"producer_attempt_count"`
|
||||||
|
TerminalAction string `json:"terminal_action"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneValidationSummary returns an independently owned durable summary.
|
||||||
|
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
|
||||||
|
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
|
||||||
|
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
|
||||||
|
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
|
||||||
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
type CheckpointDecisionManifest struct {
|
type CheckpointDecisionManifest struct {
|
||||||
@@ -163,6 +190,7 @@ type RunManifest struct {
|
|||||||
References []ReferenceProvenance `json:"references,omitempty"`
|
References []ReferenceProvenance `json:"references,omitempty"`
|
||||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||||
|
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
|
||||||
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
||||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
|||||||
@@ -116,6 +116,11 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
|
|||||||
|
|
||||||
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||||
out := in
|
out := in
|
||||||
|
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
@@ -196,6 +201,11 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
|
|||||||
|
|
||||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||||
out := in
|
out := in
|
||||||
|
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
if len(in.Options) > 0 {
|
if len(in.Options) > 0 {
|
||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
@@ -204,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := *in
|
||||||
|
if in.ProducerStructuralFailure != nil {
|
||||||
|
value := *in.ProducerStructuralFailure
|
||||||
|
out.ProducerStructuralFailure = &value
|
||||||
|
}
|
||||||
|
if in.SemanticRejection != nil {
|
||||||
|
value := *in.SemanticRejection
|
||||||
|
out.SemanticRejection = &value
|
||||||
|
}
|
||||||
|
if in.ValidatorFailure != nil {
|
||||||
|
value := *in.ValidatorFailure
|
||||||
|
out.ValidatorFailure = &value
|
||||||
|
}
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|
||||||
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||||
out := pipeline.ValidatorOverride{Set: in.Set}
|
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||||
if len(in.Validators) > 0 {
|
if len(in.Validators) > 0 {
|
||||||
|
|||||||
@@ -427,6 +427,10 @@ func (effectiveCodec) Decode(content []byte) (effectiveArtifact, error) {
|
|||||||
return value, err
|
return value, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (codec effectiveCodec) DecodeCandidate(content []byte) (effectiveArtifact, error) {
|
||||||
|
return codec.Decode(content)
|
||||||
|
}
|
||||||
|
|
||||||
type effectiveInput struct{ key string }
|
type effectiveInput struct{ key string }
|
||||||
|
|
||||||
func (m effectiveInput) Key() string { return m.key }
|
func (m effectiveInput) Key() string { return m.key }
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -35,6 +36,8 @@ type FilePromptKitLocalBackendConfig struct {
|
|||||||
|
|
||||||
type FilePipelineProfile struct {
|
type FilePipelineProfile struct {
|
||||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||||
|
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
|
||||||
Input fileModuleBinding `yaml:"input"`
|
Input fileModuleBinding `yaml:"input"`
|
||||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||||
@@ -47,15 +50,25 @@ type FilePipelineProfile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||||
|
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
type plainFilePipelineProfile FilePipelineProfile
|
type plainFilePipelineProfile FilePipelineProfile
|
||||||
var decoded plainFilePipelineProfile
|
var decoded plainFilePipelineProfile
|
||||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||||
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
"llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||||
}, "pipeline profile")
|
}, "pipeline profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
*p = FilePipelineProfile(decoded)
|
*p = FilePipelineProfile(decoded)
|
||||||
|
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
|
||||||
|
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.ValidationPolicy = policy
|
||||||
|
}
|
||||||
_, p.artifactsSet = seen["artifacts"]
|
_, p.artifactsSet = seen["artifacts"]
|
||||||
_, p.stepsSet = seen["steps"]
|
_, p.stepsSet = seen["steps"]
|
||||||
_, p.llmProfileSet = seen["llm_profile"]
|
_, p.llmProfileSet = seen["llm_profile"]
|
||||||
@@ -148,6 +161,8 @@ type FileDebugConfig struct {
|
|||||||
type fileModuleBinding struct {
|
type fileModuleBinding struct {
|
||||||
Module string
|
Module string
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
|
StructuredOutputRepairAttempts *int
|
||||||
|
ValidationPolicy *pipeline.ValidationPolicyOverride
|
||||||
Retries int
|
Retries int
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
References map[string]fileReferenceSource
|
References map[string]fileReferenceSource
|
||||||
@@ -244,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
b.Module = strings.TrimSpace(module)
|
b.Module = strings.TrimSpace(module)
|
||||||
return nil
|
return nil
|
||||||
case yaml.MappingNode:
|
case yaml.MappingNode:
|
||||||
|
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||||
for i := 0; i < len(node.Content); i += 2 {
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
keyNode := node.Content[i]
|
keyNode := node.Content[i]
|
||||||
valueNode := node.Content[i+1]
|
valueNode := node.Content[i+1]
|
||||||
|
if _, exists := seen[keyNode.Value]; exists {
|
||||||
|
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
|
||||||
|
}
|
||||||
|
seen[keyNode.Value] = struct{}{}
|
||||||
switch keyNode.Value {
|
switch keyNode.Value {
|
||||||
case "module":
|
case "module":
|
||||||
var module string
|
var module string
|
||||||
@@ -263,6 +283,18 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
if b.LLMProfile == "" {
|
if b.LLMProfile == "" {
|
||||||
return fmt.Errorf("llm_profile must not be empty when set")
|
return fmt.Errorf("llm_profile must not be empty when set")
|
||||||
}
|
}
|
||||||
|
case "structured_output_repair_attempts":
|
||||||
|
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.StructuredOutputRepairAttempts = attempts
|
||||||
|
case "validation_policy":
|
||||||
|
policy, err := parseValidationPolicy(valueNode, "module binding")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.ValidationPolicy = policy
|
||||||
case "retries":
|
case "retries":
|
||||||
var retries int
|
var retries int
|
||||||
if err := valueNode.Decode(&retries); err != nil {
|
if err := valueNode.Decode(&retries); err != nil {
|
||||||
@@ -305,13 +337,100 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
return pipeline.ModuleBinding{
|
return pipeline.ModuleBinding{
|
||||||
Module: strings.TrimSpace(b.Module),
|
Module: strings.TrimSpace(b.Module),
|
||||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
|
||||||
Retries: b.Retries,
|
Retries: b.Retries,
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
References: fileReferenceSourcesToPipeline(b.References),
|
References: fileReferenceSourcesToPipeline(b.References),
|
||||||
Validators: b.Validators,
|
Validators: cloneValidatorOverride(b.Validators),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
if node.Content[i].Value == key {
|
||||||
|
return node.Content[i+1], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
|
||||||
|
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy must be an object", context)
|
||||||
|
}
|
||||||
|
policy := &pipeline.ValidationPolicyOverride{}
|
||||||
|
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
key := node.Content[i].Value
|
||||||
|
value := node.Content[i+1]
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
|
||||||
|
}
|
||||||
|
switch key {
|
||||||
|
case "producer_structural_failure":
|
||||||
|
value := pipeline.ProducerStructuralFailureAction(value.Value)
|
||||||
|
policy.ProducerStructuralFailure = &value
|
||||||
|
case "semantic_rejection":
|
||||||
|
value := pipeline.SemanticRejectionAction(value.Value)
|
||||||
|
policy.SemanticRejection = &value
|
||||||
|
case "validator_failure":
|
||||||
|
value := pipeline.ValidatorFailureAction(value.Value)
|
||||||
|
policy.ValidatorFailure = &value
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := policy.Validate(); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
|
||||||
|
}
|
||||||
|
return policy, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
|
||||||
|
if node.Kind != yaml.MappingNode {
|
||||||
|
return fmt.Errorf("%s must be an object", context)
|
||||||
|
}
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
if node.Content[i].Value != "structured_output_repair_attempts" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
|
||||||
|
if node.Tag == "!!null" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
|
||||||
|
}
|
||||||
|
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
|
||||||
|
}
|
||||||
|
var attempts int
|
||||||
|
if err := node.Decode(&attempts); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
|
||||||
|
}
|
||||||
|
if attempts < 0 || attempts > 3 {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||||
|
}
|
||||||
|
return &attempts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
|
||||||
|
if attempts == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value := *attempts
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func LoadFileConfig(path string) (FileConfig, error) {
|
func LoadFileConfig(path string) (FileConfig, error) {
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -349,6 +468,12 @@ func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
|||||||
if err := decoder.Decode(&fileCfg); err != nil {
|
if err := decoder.Decode(&fileCfg); err != nil {
|
||||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||||
}
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); err == nil {
|
||||||
|
return FileConfig{}, fmt.Errorf("config must contain exactly one YAML document")
|
||||||
|
} else if err != io.EOF {
|
||||||
|
return FileConfig{}, fmt.Errorf("decode trailing yaml document: %w", err)
|
||||||
|
}
|
||||||
return fileCfg, nil
|
return fileCfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -513,6 +638,8 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
profile := pipeline.PipelineProfile{
|
profile := pipeline.PipelineProfile{
|
||||||
ID: pipelineID,
|
ID: pipelineID,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
|
||||||
|
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
|
||||||
Input: filePipeline.Input.toPipelineBinding(),
|
Input: filePipeline.Input.toPipelineBinding(),
|
||||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||||
|
|||||||
@@ -50,6 +50,28 @@ func TestFileConfigMinimalVersion4AppliesOverDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFileConfigYAMLRejectsAdditionalDocuments(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "trailing whitespace", source: "version: 4\n\n \t", wantErr: false},
|
||||||
|
{name: "trailing comment", source: "version: 4\n# trailing comment\n", wantErr: false},
|
||||||
|
{name: "second valid document", source: "version: 4\n---\nversion: 4\n", wantErr: true},
|
||||||
|
{name: "second empty document", source: "version: 4\n---\n", wantErr: true},
|
||||||
|
{name: "second malformed document", source: "version: 4\n---\nversion: [\n", wantErr: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(tt.source))
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want error=%t", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFilePipelineLLMProfileIsPresenceAwareAndDetached(t *testing.T) {
|
func TestFilePipelineLLMProfileIsPresenceAwareAndDetached(t *testing.T) {
|
||||||
const pipelineYAML = `version: 4
|
const pipelineYAML = `version: 4
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -116,6 +138,168 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
%s
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
validators:
|
||||||
|
- module: validator
|
||||||
|
structured_output_repair_attempts: 3
|
||||||
|
`
|
||||||
|
|
||||||
|
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
|
||||||
|
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := cfg.Pipelines["main"]
|
||||||
|
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["lane"]
|
||||||
|
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
fileProfile := file.Pipelines["main"]
|
||||||
|
fileLane := fileProfile.Artifacts["lane"]
|
||||||
|
*fileLane.Extract.StructuredOutputRepairAttempts = 1
|
||||||
|
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
|
||||||
|
fileProfile.Artifacts["lane"] = fileLane
|
||||||
|
file.Pipelines["main"] = fileProfile
|
||||||
|
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
|
||||||
|
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
|
||||||
|
}
|
||||||
|
cloned := cloneConfig(cfg)
|
||||||
|
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
redacted := cfg.Redacted()
|
||||||
|
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
summary := cfg.RedactedSummaryPayload().(Config)
|
||||||
|
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var roundTripped Config
|
||||||
|
if err := json.Unmarshal(data, &roundTripped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
roundTrippedProfile := roundTripped.Pipelines["main"]
|
||||||
|
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
|
||||||
|
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
%s
|
||||||
|
`
|
||||||
|
const bindingYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input:
|
||||||
|
module: input
|
||||||
|
%s
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
`
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
||||||
const configYAML = `version: 4
|
const configYAML = `version: 4
|
||||||
pipelines:
|
pipelines:
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user