Compare commits
42 Commits
ef19a03cbf
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 916d9210fd | |||
| 3d3f16db4a | |||
| 63c397d86a | |||
| 9a92212632 | |||
| 0ef8931697 | |||
| e00cc45c6b | |||
| ab9b743df6 | |||
| 3bd3c7ebf7 | |||
| 75a3f51cee | |||
| 2be999ebd3 | |||
| 32fe7c5b98 | |||
| 55247c47ab | |||
| 5e5c69bf9d | |||
| b4a81f8b09 | |||
| adfed22e1a | |||
| 75b1e2f68b | |||
| 4473363d9f | |||
| 6eb45e0003 | |||
| 0585ad76dc | |||
| 56145c3b7e | |||
| a478fd86c5 | |||
| 916532100d | |||
| bef8ca263b | |||
| f6d037b613 | |||
| 449b506804 | |||
| 071a78ae22 | |||
| ad1cba41c2 | |||
| 67338798aa | |||
| e95e2f2220 | |||
| 628b8d1800 | |||
| d24d4609b6 | |||
| c7f79fb38e | |||
| 8c071800cf | |||
| 569e12c6f4 | |||
| 5b6eb591b2 | |||
| b630384aa0 | |||
| 297d58f090 | |||
| ee71dc4937 | |||
| 65e5d65d14 | |||
| b40b40aaf3 | |||
| f120be1cb4 | |||
| 17673d74ea |
33
.woodpecker/release.yml
Normal file
33
.woodpecker/release.yml
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
when:
|
||||||
|
- event: tag
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: validate-release
|
||||||
|
image: golang:1.25.5
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
version="$CI_COMMIT_TAG"
|
||||||
|
release_note="docs/releases/$version.md"
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
|
||||||
|
printf '%s\n' "invalid release tag: $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$release_note" ]; then
|
||||||
|
printf '%s\n' "missing release note: $release_note" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note heading does not match $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
|
||||||
|
printf '%s\n' "release note is missing heading: $heading" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
./scripts/check-release-source.sh "$version"
|
||||||
16
README.md
16
README.md
@@ -28,6 +28,20 @@ For the complete ordered D&D workflow, use
|
|||||||
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
||||||
demonstrates all implemented D&D lanes and the supporting campaign references.
|
demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||||
|
|
||||||
|
## Install A Source Release
|
||||||
|
|
||||||
|
Install a pinned source release with Go:
|
||||||
|
|
||||||
|
~~~
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@<tag>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Replace `<tag>` with a stable release tag such as `vMAJOR.MINOR.PATCH`. The
|
||||||
|
installed command's diagnostic version is described in the [CLI
|
||||||
|
reference](docs/cli.md); maintainers preparing a release should follow [Source
|
||||||
|
Releases](docs/release.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
||||||
@@ -39,6 +53,8 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
|||||||
artifact formats.
|
artifact formats.
|
||||||
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
||||||
from an orchestrator and consume a published result.
|
from an orchestrator and consume a published result.
|
||||||
|
- [Complete D&D consumer guide](docs/consumers/dnd-pipeline.md) — run the full
|
||||||
|
D&D pipeline as a subprocess and discover its structured artifacts.
|
||||||
- [Internal overview](docs/internal/overview.md) — implemented component map
|
- [Internal overview](docs/internal/overview.md) — implemented component map
|
||||||
for maintainers.
|
for maintainers.
|
||||||
- [Developer guide](docs/development.md) — contributor orientation and
|
- [Developer guide](docs/development.md) — contributor orientation and
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_combat_turns_llm.v1.json
|
schema_path: dnd_combat_turns_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -50,4 +50,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_enemy_events_llm.v1.json
|
schema_path: dnd_enemy_events_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
||||||
"$id": "notarius.dnd.entity_reconcile.llm",
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["duplicate_groups"],
|
|
||||||
"properties": {
|
|
||||||
"duplicate_groups": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["members", "canonical"],
|
|
||||||
"properties": {
|
|
||||||
"members": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"$ref": "#/$defs/selector"}
|
|
||||||
},
|
|
||||||
"canonical": {"$ref": "#/$defs/selector"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"$defs": {
|
|
||||||
"selector": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["name", "source_refs"],
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string", "minLength": 1},
|
|
||||||
"source_refs": {
|
|
||||||
"type": "array",
|
|
||||||
"minItems": 1,
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["start_unit_id", "end_unit_id"],
|
|
||||||
"properties": {
|
|
||||||
"start_unit_id": {"type": "integer", "minimum": 1},
|
|
||||||
"end_unit_id": {"type": "integer", "minimum": 1}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_occurrences_llm.v1.json
|
schema_path: dnd_item_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_item_registry_llm.v1.json
|
schema_path: dnd_item_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
Use candidate names and cited transcript windows only to determine whether
|
Determine whether candidates identify the same item type or unique designation
|
||||||
candidates identify the same item type or unique designation. Do not treat
|
using their contextual labels and cited transcript windows. Do not treat nearby
|
||||||
nearby evidence, similar objects, or a shared owner as sufficient. Keep
|
evidence, similar objects, or a shared owner as sufficient.
|
||||||
currency denominations, materially different item types, and uncertain aliases
|
|
||||||
separate. Do not infer an item property or uniqueness.
|
Keep currency denominations and materially different item types separate. Keep
|
||||||
|
uncertain aliases separate. Do not infer an item property or uniqueness.
|
||||||
|
|
||||||
When selecting a canonical display name, choose one supplied candidate name
|
When selecting a canonical display name, choose one supplied candidate name
|
||||||
that is the clearest established designation.
|
that is the clearest established designation.
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_occurrences_llm.v1.json
|
schema_path: dnd_location_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_location_registry_llm.v1.json
|
schema_path: dnd_location_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
Location candidates:
|
|
||||||
{{ input "candidates" }}
|
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
Use candidate names and their cited transcript windows to determine whether
|
Determine whether candidates identify the same physical place using their
|
||||||
candidates identify the same physical place. Do not treat matching names,
|
contextual labels and cited transcript windows. Do not treat matching names,
|
||||||
nearby evidence, nested places, or generic labels as sufficient. Keep parent
|
nearby evidence, nested places, or generic labels as sufficient.
|
||||||
and child places, similarly named places, and uncertain aliases separate.
|
|
||||||
|
Keep parent and child places separate, as well as similarly named places and
|
||||||
|
uncertain aliases.
|
||||||
|
|
||||||
When selecting a canonical display name, prefer the clearest established name.
|
When selecting a canonical display name, prefer the clearest established name.
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_npc_registry_llm.v1.json
|
schema_path: dnd_npc_registry_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
NPC candidates for identity comparison:
|
|
||||||
|
|
||||||
{{ input "candidates" }}
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
Use candidate aliases and their cited transcript windows to determine whether
|
Determine whether candidates refer to the same individual using their
|
||||||
candidates refer to the same individual. Preserve distinct individuals even
|
contextual labels and cited transcript windows. Preserve distinct individuals
|
||||||
when their names are similar.
|
even when their names are similar or their contextual descriptions are
|
||||||
|
identical.
|
||||||
|
|
||||||
When selecting a canonical display name, prefer a complete, stable proper name
|
When selecting a canonical display name, prefer a complete, stable proper name
|
||||||
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
||||||
|
|||||||
@@ -12,19 +12,19 @@ messages:
|
|||||||
- role: system
|
- role: system
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./sharedassets/protocol.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-entity-reconciliation.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./candidates.md
|
content_file: ./sharedassets/candidates.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./sharedassets/common-dnd-transcript-windows.md
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_entity_reconcile_llm.v1.json
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -35,4 +35,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -31,4 +31,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_scenes_llm.v1.json
|
schema_path: dnd_scenes_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
Identify only well-supported duplicate groups among the supplied candidates.
|
|
||||||
|
|
||||||
Return each selected candidate's supplied contextual descriptor exactly: its
|
|
||||||
`name` and complete ordered `source_refs`. A group must contain at least two
|
|
||||||
supplied descriptors, and its `canonical` descriptor must be one of its
|
|
||||||
members. Do not invent names, ranges, records, evidence, or replacement values.
|
|
||||||
Omit any uncertain or unsafe group.
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
Transcript units are the only evidence for extracted events and factual claims.
|
Transcript units are the only evidence for extracted events and factual claims.
|
||||||
Every reported factual claim must be supported by cited transcript units. Use
|
Every reported factual claim must be supported by cited transcript units. Use
|
||||||
integer `start_unit_id` and `end_unit_id` values from the transcript. Omit
|
integer `start_unit_id` and `end_unit_id` values from the transcript.
|
||||||
`source_id`; Notarius assigns the current source identity.
|
|
||||||
|
|
||||||
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
||||||
than a broad range that bridges unrelated conversation.
|
than a broad range that bridges unrelated conversation.
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
You process Dungeons & Dragons gameplay transcripts.
|
You process Dungeons & Dragons gameplay transcripts.
|
||||||
|
|
||||||
Rely only on the supplied inputs. They may contain transcription errors,
|
As input, you will receive one or more portions of a transcript. The transcript may contain transcription errors, repeated lines, incomplete sentences, and misheard proper nouns.
|
||||||
repeated lines, incomplete sentences, and misheard proper nouns.
|
|
||||||
|
|
||||||
Return exactly one JSON object that conforms to the configured response schema,
|
Return exactly one JSON object that conforms to the configured response schema, with no explanatory prose.
|
||||||
with no explanatory prose.
|
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
One extraction chunk from a Dungeons & Dragons gameplay transcript is provided
|
One extraction chunk from a Dungeons & Dragons gameplay transcript is provided below. Report and infer only what is within this chunk. Its unit IDs retain their source-wide meaning.
|
||||||
below. Report and infer only what is within this chunk. Its unit IDs retain
|
|
||||||
their source-wide meaning.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
{{ input "transcript" }}
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
The complete ordered transcript of this Dungeons & Dragons gameplay session is
|
The complete ordered transcript of this Dungeons & Dragons gameplay session is provided below.
|
||||||
provided below. It may contain multiple scenes.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
{{ input "transcript" }}
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
Selected Dungeons & Dragons gameplay transcript evidence windows are provided
|
|
||||||
below. They may be incomplete, non-contiguous, or overlapping. Use them to
|
|
||||||
evaluate candidate identity, but do not treat absence outside these windows as
|
|
||||||
evidence.
|
|
||||||
|
|
||||||
{{ input "transcript" }}
|
|
||||||
@@ -47,4 +47,4 @@ output:
|
|||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: dnd_spells_llm.v1.json
|
schema_path: dnd_spells_llm.v1.json
|
||||||
repair_attempts: 0
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
Item candidates:
|
Candidate material:
|
||||||
|
|
||||||
{{ input "candidates" }}
|
{{ input "candidates" }}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
Identify only high-confidence duplicate entities among the supplied candidates.
|
||||||
|
|
||||||
|
Preserve distinct entities even when their names are similar. Treat contextual descriptions and transcript evidence as supporting material, not as permission to merge ambiguous records.
|
||||||
|
|
||||||
|
When several records are duplicates, choose as canonical the candidate with the clearest stable identity. Prefer a complete proper name over an abbreviation, and prefer an unadorned proper name over one with incidental descriptors unless the evidence establishes those descriptors as part of the name. A longer name is not inherently more canonical.
|
||||||
27
assets/generic/normalize/deduplication/prompts/prompt.yaml
Normal file
27
assets/generic/normalize/deduplication/prompts/prompt.yaml
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
id: generic.semantic_reconciliation
|
||||||
|
version: "v1"
|
||||||
|
inputs:
|
||||||
|
- name: candidates
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content_file: ./system.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./protocol.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./instructions.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./candidates.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./transcript-windows.md
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Use only the positive integer `candidate_id` values supplied in the candidate material.
|
||||||
|
|
||||||
|
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group.
|
||||||
|
|
||||||
|
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid.
|
||||||
|
|
||||||
|
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||||
2
assets/generic/normalize/deduplication/prompts/system.md
Normal file
2
assets/generic/normalize/deduplication/prompts/system.md
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
You reconcile structured records that may describe the same underlying entity.
|
||||||
|
Follow the supplied protocol and return only the requested structured result.
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
Transcript evidence windows:
|
||||||
|
|
||||||
|
{{ input "transcript" }}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "notarius.generic.semantic_reconciliation.llm",
|
||||||
|
"title": "notarius_semantic_reconciliation_llm_v1",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["duplicate_groups"],
|
||||||
|
"properties": {
|
||||||
|
"duplicate_groups": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["candidate_ids", "canonical_candidate_id"],
|
||||||
|
"properties": {
|
||||||
|
"candidate_ids": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 2,
|
||||||
|
"items": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"canonical_candidate_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
# ADR-0013: Use request-local candidate handles for semantic reconciliation
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-08-09
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Several typed normalize stage modules need semantic reconciliation after
|
||||||
|
deterministic preprocessing: a model can judge whether source-backed candidates
|
||||||
|
refer to the same underlying entity, while application code remains responsible
|
||||||
|
for constructing the normalized artifact. Requiring the model to reproduce a
|
||||||
|
candidate's full contextual selector makes the response larger and introduces
|
||||||
|
avoidable formatting, ordering, and transcription failure modes.
|
||||||
|
|
||||||
|
Reconciliation must preserve the exact typed artifact boundary established by
|
||||||
|
[ADR-0003](0003-typed-interfaces-with-two-zone-data-model.md), the domain-neutral
|
||||||
|
framework and concrete-domain dependency direction established by
|
||||||
|
[ADR-0004](0004-package-modules-by-domain.md), and the distinction in
|
||||||
|
[ADR-0009](0009-minimal-evidence-grounded-extraction-artifacts.md) between source
|
||||||
|
evidence and auxiliary identity context. It also needs a concrete, narrowly
|
||||||
|
scoped application of the request-local-label exception allowed by
|
||||||
|
[ADR-0012](0012-resolve-opaque-entity-identifiers-deterministically.md).
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Semantic reconciliation will be a domain-neutral framework mechanism used by
|
||||||
|
typed normalize stage modules. A consuming artifact family will retain
|
||||||
|
ownership of its typed records, identity rules, consolidation policy, durable
|
||||||
|
IDs, and domain warnings; the framework mechanism will not infer those rules
|
||||||
|
from arbitrary data.
|
||||||
|
|
||||||
|
For each reconciliation request, deterministic code will assign every eligible
|
||||||
|
model-visible candidate a contiguous, one-based integer handle. The model may
|
||||||
|
receive the candidate's contextual label, source references, and bounded source
|
||||||
|
context needed to judge identity, but its structured response will identify
|
||||||
|
candidates only by those supplied handles. A handle is local to one request,
|
||||||
|
does not represent entity identity, and must never enter a durable artifact or
|
||||||
|
be used to derive a durable ID.
|
||||||
|
|
||||||
|
The model will propose duplicate groups and select one supplied member of each
|
||||||
|
group as canonical. Deterministic code will resolve the handles through the
|
||||||
|
retained request mapping, validate the complete proposal, discard unsafe
|
||||||
|
groups, and apply only validated groups through typed domain-owned policy. The
|
||||||
|
model will not synthesize replacement records or directly mutate an artifact.
|
||||||
|
|
||||||
|
Every reconciliation prompt will combine a mandatory framework-owned protocol
|
||||||
|
and safety policy with an explicitly selected semantic policy. The semantic
|
||||||
|
policy may be the conservative generic policy or a domain-owned policy, but it
|
||||||
|
cannot replace the shared response protocol or deterministic safety boundary.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Return durable application IDs. Opaque IDs do not help semantic judgment,
|
||||||
|
expose application identity mechanics, and make model output reproduce data
|
||||||
|
that deterministic code already owns.
|
||||||
|
- Return names alone or copied contextual selectors. Names can be ambiguous,
|
||||||
|
while reproducing labels and source ranges adds response complexity and
|
||||||
|
creates mismatches without adding semantic information. Request-local
|
||||||
|
handles preserve exact selection without either failure mode.
|
||||||
|
- Ask the model to return synthesized canonical replacement records. This
|
||||||
|
would transfer typed artifact construction, provenance consolidation, and
|
||||||
|
durable identity policy to a probabilistic boundary.
|
||||||
|
- Reconcile reflection-discovered fields or arbitrary JSON. This would weaken
|
||||||
|
the typed artifact contract and move domain semantics into generic code.
|
||||||
|
- Hide reconciliation inside extraction or another stage. This would obscure
|
||||||
|
stage ownership and create cross-stage behavior outside the fixed pipeline;
|
||||||
|
reconciliation remains explicit normalize-stage behavior.
|
||||||
|
- Let each domain replace the complete prompt protocol. This would duplicate
|
||||||
|
safety mechanics and allow domain policy to bypass the common response and
|
||||||
|
validation contract.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Model responses become smaller and easier to validate, while deterministic
|
||||||
|
application code retains authority over identity, provenance, ordering, and
|
||||||
|
typed artifact construction. The framework requires a request-local mapping,
|
||||||
|
bounded context preparation, a private integer response contract, proposal
|
||||||
|
assessment, and shared prompt assets. Each consuming artifact family still
|
||||||
|
requires a typed adapter for its irreducibly domain-specific rules.
|
||||||
|
|
||||||
|
Request-local handles are deliberately unsuitable for persistence, logging as
|
||||||
|
entity identity, checkpoint contracts, or cross-request correlation. Changes
|
||||||
|
to shared protocol and policy assets must participate in the normal prompt,
|
||||||
|
schema, and checkpoint fingerprint mechanisms.
|
||||||
|
|
||||||
|
Acceptance of this decision does not imply that the shared mechanism or its
|
||||||
|
consumer migrations are implemented. The
|
||||||
|
[feature roadmap](../roadmap/semantic-reconciliation.md) owns target behavior
|
||||||
|
and status, and the
|
||||||
|
[implementation plan](../roadmap/implementation.md) owns delivery sequence
|
||||||
|
until the work is complete.
|
||||||
12
docs/cli.md
12
docs/cli.md
@@ -10,6 +10,7 @@ defined in [Operations](operations.md).
|
|||||||
|
|
||||||
~~~
|
~~~
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||||
@@ -18,6 +19,17 @@ notarius pipelines list [--config path/to/config.yml] [--json]
|
|||||||
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
||||||
writes the command summary to standard output and exits with status 0.
|
writes the command summary to standard output and exits with status 0.
|
||||||
|
|
||||||
|
`notarius --version` is valid only as the sole root argument. It writes exactly
|
||||||
|
`notarius <version>` followed by a newline to standard output and exits with
|
||||||
|
status 0. A tagged `go install` build can report its main-module stable tag,
|
||||||
|
and controlled builds can inject a stable tag at link time through
|
||||||
|
`gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override`; an ordinary
|
||||||
|
unversioned checkout reports `development`. Invalid injected version content is
|
||||||
|
a runtime error with exit status 1, while extra `--version` arguments are a
|
||||||
|
syntax error with exit status 2. This diagnostic does not replace the
|
||||||
|
[run-result](integrations/run-result.md) or artifact contracts for downstream
|
||||||
|
compatibility decisions.
|
||||||
|
|
||||||
## run
|
## run
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
|
|||||||
@@ -132,7 +132,11 @@ model: example-model
|
|||||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||||
its credential environment variable through `api_key_env`; set that variable
|
its credential environment variable through `api_key_env`; set that variable
|
||||||
only in the run environment. PromptKit owns the
|
only in the run environment. PromptKit owns the
|
||||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md),
|
||||||
|
including `base_profile` inheritance. Notarius passes profiles through without
|
||||||
|
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||||
|
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||||
|
provider without authorization.
|
||||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||||
supported package API, and [Operations](operations.md#operational-limits)
|
supported package API, and [Operations](operations.md#operational-limits)
|
||||||
describes the effective concurrency layers.
|
describes the effective concurrency layers.
|
||||||
@@ -218,6 +222,7 @@ pipelines:
|
|||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||||
|
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
|
||||||
| **input** | module binding | none | Required. |
|
| **input** | module binding | none | Required. |
|
||||||
| **chunk** | module binding | **generic** | Optional. |
|
| **chunk** | module binding | **generic** | Optional. |
|
||||||
| **output** | module binding | **json** | Optional. |
|
| **output** | module binding | **json** | Optional. |
|
||||||
@@ -237,6 +242,15 @@ run-level **--llm-profile** value first, then the binding's **llm_profile**,
|
|||||||
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
||||||
Deterministic bindings do not receive these defaults or run overrides.
|
Deterministic bindings do not receive these defaults or run overrides.
|
||||||
|
|
||||||
|
Structural output repair is resolved after module, validator, and `--only` lane
|
||||||
|
selection. An object's **structured_output_repair_attempts** value takes
|
||||||
|
precedence over the pipeline value; otherwise, an LLM-backed binding or
|
||||||
|
validator inherits the pipeline value. If both are omitted, PromptKit uses the
|
||||||
|
prompt's declared repair policy. The value must be an integer from 0 through 3;
|
||||||
|
explicit `null` and non-integer values are invalid. An explicit value on a
|
||||||
|
deterministic binding or validator is invalid, while a pipeline value simply
|
||||||
|
does not apply to deterministic selections.
|
||||||
|
|
||||||
A lane has these fields:
|
A lane has these fields:
|
||||||
|
|
||||||
| Field | Type | Default | Rules |
|
| Field | Type | Default | Rules |
|
||||||
@@ -274,6 +288,7 @@ extract:
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||||
|
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
|
||||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
||||||
| **options** | object | none | Must satisfy the selected module. |
|
| **options** | object | none | Must satisfy the selected module. |
|
||||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||||
@@ -281,10 +296,11 @@ extract:
|
|||||||
|
|
||||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||||
Validator bindings accept only **module**, **llm_profile**, and **options**.
|
Validator bindings accept only **module**, **llm_profile**,
|
||||||
They reject **references**, **retries**, and nested **validators**. Deterministic
|
**structured_output_repair_attempts**, and **options**. They reject
|
||||||
validators reject an explicit **llm_profile**. Deterministic module bindings
|
**references**, **retries**, and nested **validators**. Deterministic validators
|
||||||
also reject an explicit **llm_profile**.
|
reject explicit **llm_profile** and **structured_output_repair_attempts**.
|
||||||
|
Deterministic module bindings also reject those explicit fields.
|
||||||
|
|
||||||
The **json** output module accepts optional **include_chunk_map** and
|
The **json** output module accepts optional **include_chunk_map** and
|
||||||
**evidence_context** settings:
|
**evidence_context** settings:
|
||||||
@@ -319,7 +335,8 @@ Unknown outer or nested option fields are rejected, as are incompatible YAML
|
|||||||
types. The allowlist remains valid when a run uses lane filtering: a configured
|
types. The allowlist remains valid when a run uses lane filtering: a configured
|
||||||
lane that is not active for that invocation simply contributes no evidence.
|
lane that is not active for that invocation simply contributes no evidence.
|
||||||
Evidence publication is opt-in because it can persist source text and metadata.
|
Evidence publication is opt-in because it can persist source text and metadata.
|
||||||
Its payload contract is [Published Evidence Context](integrations/evidence-context.md).
|
When enabled, it publishes the selected source-unit excerpt defined by the
|
||||||
|
[Published Evidence Context contract](integrations/evidence-context.md).
|
||||||
|
|
||||||
## References And Ordered Handoffs
|
## References And Ordered Handoffs
|
||||||
|
|
||||||
|
|||||||
202
docs/consumers/dnd-pipeline.md
Normal file
202
docs/consumers/dnd-pipeline.md
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
# Consuming The Complete D&D Pipeline
|
||||||
|
|
||||||
|
Use this workflow when an orchestrator runs the maintained complete D&D
|
||||||
|
pipeline and consumes its structured JSON artifacts. The generic
|
||||||
|
[subprocess consumer guide](subprocess.md) owns process-level responsibilities;
|
||||||
|
this guide connects that workflow to the complete D&D configuration, its
|
||||||
|
Seriatim input, and its artifact inventory.
|
||||||
|
|
||||||
|
The [CLI reference](../cli.md), [configuration reference](../config.md),
|
||||||
|
[run-result receipt](../integrations/run-result.md), and
|
||||||
|
[published JSON output contract](../integrations/json-output.md) remain the
|
||||||
|
canonical definitions of those public interfaces.
|
||||||
|
|
||||||
|
## Prepare And Validate The Deployment
|
||||||
|
|
||||||
|
Start from the maintained
|
||||||
|
[complete D&D configuration](../../examples/dnd-complete.config.yml). It uses
|
||||||
|
the `dnd-session` pipeline and demonstrates every implemented D&D lane, ordered
|
||||||
|
artifact handoffs, campaign references, chunk-map publication, and evidence
|
||||||
|
context.
|
||||||
|
|
||||||
|
A deployment must provide its own PromptKit profile and campaign reference
|
||||||
|
files. Use absolute paths for service and subprocess deployments. In
|
||||||
|
particular, observe these different resolution rules:
|
||||||
|
|
||||||
|
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||||
|
file; and
|
||||||
|
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||||
|
directory.
|
||||||
|
|
||||||
|
Do not copy the repository example's relative profile path into a deployment
|
||||||
|
without also controlling that working directory. The complete path and profile
|
||||||
|
rules are defined in [Configuration](../config.md).
|
||||||
|
|
||||||
|
Preflight the deployed configuration before processing sessions and whenever
|
||||||
|
it changes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius config validate \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--pipeline dnd-session
|
||||||
|
```
|
||||||
|
|
||||||
|
Provide credentials through the environment or the documented configuration
|
||||||
|
mechanism. Do not put credentials in command arguments, generated
|
||||||
|
configuration, or logs.
|
||||||
|
|
||||||
|
## Supply The Transcript
|
||||||
|
|
||||||
|
The complete pipeline consumes a Seriatim JSON document. The
|
||||||
|
[Seriatim input contract](../integrations/seriatim.md) defines its required
|
||||||
|
metadata, segments, and validation rules. Preserve segment IDs: D&D artifact
|
||||||
|
citations use those segment IDs as source-unit ranges.
|
||||||
|
|
||||||
|
When the caller maintains several transcript tiers, use the final trimmed JSON
|
||||||
|
transcript so extraction operates on the same session content presented to
|
||||||
|
later consumers. For example, Narratio identifies this implemented artifact as
|
||||||
|
`narratio.transcript.final_trimmed` and normally stores it at
|
||||||
|
`transcripts/final.trimmed.json`.
|
||||||
|
|
||||||
|
Notarius generates a stable prompt session from the resolved input module and
|
||||||
|
the exact input bytes. An ordinary orchestrator should not pass `--session-id`.
|
||||||
|
Use that override only when intentionally changing the routing relationship
|
||||||
|
between invocations; it is not a credential or output identity.
|
||||||
|
|
||||||
|
## Run Notarius
|
||||||
|
|
||||||
|
Invoke the pipeline with explicit absolute paths and request its
|
||||||
|
machine-readable receipt:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius run dnd-session \
|
||||||
|
--config /absolute/path/to/notarius.yml \
|
||||||
|
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||||
|
--output-dir /absolute/path/to/notarius-output \
|
||||||
|
--json
|
||||||
|
```
|
||||||
|
|
||||||
|
The caller should:
|
||||||
|
|
||||||
|
- capture stdout and stderr separately;
|
||||||
|
- propagate cancellation and impose an operator-appropriate timeout;
|
||||||
|
- wait for process completion before interpreting stdout; and
|
||||||
|
- retain stderr for diagnosis without copying secrets or transcript content
|
||||||
|
into other logs.
|
||||||
|
|
||||||
|
Only exit status 0 permits decoding stdout as a receipt. Ignore stdout after a
|
||||||
|
nonzero exit because a failed receipt write can leave partial bytes. The
|
||||||
|
[CLI reference](../cli.md#output-streams-and-exit-statuses) defines the complete
|
||||||
|
stream and exit-status contract.
|
||||||
|
|
||||||
|
## Discover The Published Bundle
|
||||||
|
|
||||||
|
Decode the successful stdout document as a supported run-result schema. For
|
||||||
|
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate
|
||||||
|
unknown fields allowed by that version, but reject an unsupported schema
|
||||||
|
version.
|
||||||
|
|
||||||
|
Use the receipt's absolute `output_directory` as the exact run-specific bundle
|
||||||
|
root. Do not scan the output root for its newest directory, guess a run ID, or
|
||||||
|
construct a bundle path. Resolve `index_file` beneath `output_directory` and
|
||||||
|
reject an absolute logical path or any result that escapes the bundle root.
|
||||||
|
|
||||||
|
Read `index.json` and locate each requested lane in `output_files` by its exact
|
||||||
|
`lane_id`. Do not guess a lane filename. Before decoding a payload:
|
||||||
|
|
||||||
|
1. resolve its descriptor's relative `file` beneath the bundle root with the
|
||||||
|
same confinement check;
|
||||||
|
2. verify the descriptor's media type and schema identity against the linked
|
||||||
|
artifact contract; and
|
||||||
|
3. decode the payload according to that contract.
|
||||||
|
|
||||||
|
The [published JSON output contract](../integrations/json-output.md) defines
|
||||||
|
the index and bundle layout. Treat all paths obtained from a decoded external
|
||||||
|
document as untrusted until confined to their documented root.
|
||||||
|
|
||||||
|
## Complete Artifact Inventory
|
||||||
|
|
||||||
|
When every configured lane is accepted, the complete example publishes these
|
||||||
|
lane artifacts:
|
||||||
|
|
||||||
|
| Lane ID | Purpose | Canonical contract |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `item-registry` | Canonical registry of encountered items and currency. | [Item registry](../integrations/dnd-item-registry-artifacts.md) |
|
||||||
|
| `npc-registry` | Canonical registry of named NPCs. | [NPC registry](../integrations/dnd-npc-registry-artifacts.md) |
|
||||||
|
| `location-registry` | Canonical registry of named locations. | [Location registry](../integrations/dnd-location-registry-artifacts.md) |
|
||||||
|
| `scene-descriptions` | Classification, title, and summary for each scene. | [Scene descriptions](../integrations/dnd-scene-description-artifacts.md) |
|
||||||
|
| `item-occurrences` | Source-grounded item discovery, acquisition, use, transfer, and loss events. | [Item occurrences](../integrations/dnd-item-occurrence-artifacts.md) |
|
||||||
|
| `spells` | Source-grounded spell casts and casters. | [Spell casts](../integrations/dnd-spell-artifacts.md) |
|
||||||
|
| `combat-turns` | Source-grounded combat turn participation. | [Combat turns](../integrations/dnd-combat-turn-artifacts.md) |
|
||||||
|
| `npc-occurrences` | Source-grounded NPC interaction occurrences. | [NPC occurrences](../integrations/dnd-npc-occurrence-artifacts.md) |
|
||||||
|
| `location-occurrences` | Source-grounded location occurrences. | [Location occurrences](../integrations/dnd-location-occurrence-artifacts.md) |
|
||||||
|
| `enemy-events` | Source-grounded enemy combat events. | [Enemy events](../integrations/dnd-enemy-event-artifacts.md) |
|
||||||
|
|
||||||
|
The JSON encoder always publishes these bundle-management files:
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
|
||||||
|
| `manifest.json` | Run provenance and result summaries. |
|
||||||
|
| `rejected.json` | Rejected pipeline outputs. |
|
||||||
|
| `warnings.json` | Accepted-output and run warnings. |
|
||||||
|
|
||||||
|
The complete configuration also requests two pipeline-wide artifacts:
|
||||||
|
|
||||||
|
- [`chunk-map.json`](../integrations/chunk-map.md), the accepted chunk plan and
|
||||||
|
chunk metadata; and
|
||||||
|
- [`evidence-context.json`](../integrations/evidence-context.md), a reading
|
||||||
|
excerpt containing the union of selected cited source units and the
|
||||||
|
configured surrounding window.
|
||||||
|
|
||||||
|
Discover both from their top-level `index.json` descriptors rather than
|
||||||
|
treating them as lanes. Evidence context is convenient reading material, not
|
||||||
|
authoritative provenance; citations in the normalized lane payloads remain the
|
||||||
|
evidence contract.
|
||||||
|
|
||||||
|
Every optional or lane file is published only when its corresponding artifact
|
||||||
|
is available. A successful process does not guarantee that all configured
|
||||||
|
lanes were accepted.
|
||||||
|
|
||||||
|
## Decide What Counts As Consumer Success
|
||||||
|
|
||||||
|
Exit status 0 means Notarius completed the pipeline and published its result
|
||||||
|
bundle. The receipt or bundle may still report warnings, rejected outputs, or
|
||||||
|
missing lane descriptors. A downstream consumer must define its own required
|
||||||
|
artifact set explicitly.
|
||||||
|
|
||||||
|
A caller that claims to consume the complete D&D workflow should normally
|
||||||
|
require all ten lane IDs in the table and verify each descriptor's expected
|
||||||
|
contract. If any required lane is missing, rejected, or incompatible, fail the
|
||||||
|
caller's extraction step while retaining the Notarius bundle for diagnosis. A
|
||||||
|
consumer that needs only a subset may define and document a narrower policy.
|
||||||
|
|
||||||
|
Keep the successful receipt with the complete published bundle. Retain
|
||||||
|
`manifest.json`, `rejected.json`, `warnings.json`, and captured process logs as
|
||||||
|
required by the caller's provenance, diagnosis, and retention policies. Avoid
|
||||||
|
selectively copying payload files without also preserving enough index and
|
||||||
|
manifest information to identify their originating run and contracts.
|
||||||
|
|
||||||
|
The transcript, lane artifacts, evidence context, manifest, debug data, and
|
||||||
|
logs can all contain private campaign information. Apply the same access,
|
||||||
|
publication, and retention controls used for the source transcript.
|
||||||
|
|
||||||
|
## Consumer Checklist
|
||||||
|
|
||||||
|
- Validate the deployed Notarius configuration and `dnd-session` pipeline.
|
||||||
|
- Pass the final trimmed Seriatim JSON transcript with stable segment IDs.
|
||||||
|
- Use absolute configuration, input, output-root, profile, and reference paths
|
||||||
|
in service deployments.
|
||||||
|
- Capture stdout and stderr separately and enforce cancellation and timeout.
|
||||||
|
- Parse stdout only after exit status 0.
|
||||||
|
- Accept only supported receipt, index, and artifact schema versions while
|
||||||
|
tolerating permitted unknown fields.
|
||||||
|
- Use the receipt's `output_directory`; never guess the run directory.
|
||||||
|
- Confine `index_file` and every descriptor path to the published bundle root.
|
||||||
|
- Discover lanes by `lane_id` and verify descriptor compatibility before
|
||||||
|
decoding payloads.
|
||||||
|
- Enforce an explicit required-lane policy and inspect rejections and warnings.
|
||||||
|
- Preserve the receipt and sufficient bundle provenance for every retained
|
||||||
|
artifact.
|
||||||
|
- Protect all transcript-derived files and diagnostic streams as sensitive
|
||||||
|
campaign data.
|
||||||
@@ -6,6 +6,10 @@ statuses, while the [run-result receipt](../integrations/run-result.md) and
|
|||||||
[Published JSON Output contract](../integrations/json-output.md) own the
|
[Published JSON Output contract](../integrations/json-output.md) own the
|
||||||
durable result formats.
|
durable result formats.
|
||||||
|
|
||||||
|
For the maintained complete D&D workflow, including its transcript input,
|
||||||
|
configured lane inventory, and downstream acceptance checklist, see
|
||||||
|
[Consuming The Complete D&D Pipeline](dnd-pipeline.md).
|
||||||
|
|
||||||
## Run And Check The Process
|
## Run And Check The Process
|
||||||
|
|
||||||
Optionally preflight a selected configuration and pipeline before work starts:
|
Optionally preflight a selected configuration and pipeline before work starts:
|
||||||
@@ -55,10 +59,10 @@ contract. The JSON bundle contract links to the available lane contracts.
|
|||||||
If `index.json` has an `evidence_context` descriptor, treat it as a
|
If `index.json` has an `evidence_context` descriptor, treat it as a
|
||||||
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
||||||
fields before decoding the linked file according to the [Published Evidence
|
fields before decoding the linked file according to the [Published Evidence
|
||||||
Context contract](../integrations/evidence-context.md). Use each
|
Context contract](../integrations/evidence-context.md). Decode its top-level
|
||||||
`evidence_refs` entry as the citation to source material. Its surrounding
|
source-unit array as a reading excerpt. Obtain authoritative citations and lane
|
||||||
context range and included units explain the citation, but do not widen or
|
provenance from the normalized lane artifacts; the excerpt has neither and its
|
||||||
replace the cited source reference.
|
nearby units do not widen a lane artifact's cited source reference.
|
||||||
|
|
||||||
A zero exit status may still report rejected outputs, warnings, or absent
|
A zero exit status may still report rejected outputs, warnings, or absent
|
||||||
lanes. The caller decides which lane IDs are required for its own work and
|
lanes. The caller decides which lane IDs are required for its own work and
|
||||||
@@ -73,5 +77,5 @@ them. Treat the input, output bundle, cache, debug bundle, and captured process
|
|||||||
logs as potentially sensitive data. Apply the caller's access controls and
|
logs as potentially sensitive data. Apply the caller's access controls and
|
||||||
retention policy, and avoid copying secrets into arguments, logs, or
|
retention policy, and avoid copying secrets into arguments, logs, or
|
||||||
provenance records. An evidence-context artifact contains source-unit text and
|
provenance records. An evidence-context artifact contains source-unit text and
|
||||||
metadata, and selected lanes can cover most of an input; preserve and share it
|
metadata and can cover most of an input; preserve and share it only when that
|
||||||
only when that source content is authorized for the recipient.
|
source content is authorized for the recipient.
|
||||||
|
|||||||
@@ -18,13 +18,14 @@ implemented component map.
|
|||||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||||
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
||||||
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
||||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
|
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Complete D&D Consumer Guide](consumers/dnd-pipeline.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate generic caller workflow, the complete D&D workflow, the durable receipt contract, and CLI implementation behavior. |
|
||||||
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
||||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||||
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
||||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
||||||
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||||
|
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
|
||||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||||
|
|
||||||
For an existing subsystem, also inspect its focused tests and the package-local
|
For an existing subsystem, also inspect its focused tests and the package-local
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# Published Evidence Context
|
# Published Evidence Context
|
||||||
|
|
||||||
This contract defines the optional `source/evidence-context` artifact emitted
|
This contract defines the optional `source/evidence-context` artifact emitted
|
||||||
by the production JSON output. Its configuration is owned by
|
by the production JSON output. It is a selected source-unit excerpt for
|
||||||
[Configuration](../config.md#module-bindings-and-validators); its logical-file
|
convenient reading alongside normalized lane artifacts; it is not a second
|
||||||
discovery is owned by [Published JSON Output](json-output.md).
|
citation or provenance model. Its configuration is owned by
|
||||||
|
[Configuration](../config.md#module-bindings-and-validators), and its
|
||||||
|
logical-file discovery is owned by [Published JSON Output](json-output.md).
|
||||||
|
|
||||||
## Identity And Discovery
|
## Identity And Discovery
|
||||||
|
|
||||||
@@ -26,91 +28,80 @@ its absence means evidence publication was not enabled for that bundle.
|
|||||||
|
|
||||||
## Payload
|
## Payload
|
||||||
|
|
||||||
The v1 payload is a JSON object with required `source_id`, `source_digest`,
|
The v1 payload is a top-level JSON array of generic source units. There is no
|
||||||
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
|
wrapper, source-level metadata, context grouping, lane identifier, or evidence
|
||||||
`contexts` are always arrays; an enabled configuration with no accepted direct
|
reference in the payload. An enabled configuration with no contributing
|
||||||
evidence publishes `contexts: []`.
|
accepted evidence publishes `[]`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
[
|
||||||
"source_id": "session-alpha",
|
{
|
||||||
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
"id": 10,
|
||||||
"window_units": 1,
|
"kind": "transcript_segment",
|
||||||
"selected_lanes": ["npc_registry", "spells"],
|
"text": "Aria casts Cure Wounds.",
|
||||||
"contexts": [
|
"ref": {
|
||||||
{
|
"source_id": "session-alpha",
|
||||||
"context_ref": {
|
"start_unit_id": 10,
|
||||||
"source_id": "session-alpha",
|
"end_unit_id": 10
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 20
|
|
||||||
},
|
|
||||||
"evidence_refs": [
|
|
||||||
{
|
|
||||||
"lane_id": "spells",
|
|
||||||
"source_ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"units": [
|
|
||||||
{
|
|
||||||
"id": 10,
|
|
||||||
"kind": "transcript_segment",
|
|
||||||
"text": "Aria casts Cure Wounds.",
|
|
||||||
"ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 10,
|
|
||||||
"end_unit_id": 10
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 20,
|
|
||||||
"kind": "transcript_segment",
|
|
||||||
"text": "The party regroups.",
|
|
||||||
"ref": {
|
|
||||||
"source_id": "session-alpha",
|
|
||||||
"start_unit_id": 20,
|
|
||||||
"end_unit_id": 20
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
]
|
},
|
||||||
}
|
{
|
||||||
|
"id": 20,
|
||||||
|
"kind": "transcript_segment",
|
||||||
|
"text": "The party regroups.",
|
||||||
|
"ref": {
|
||||||
|
"source_id": "session-alpha",
|
||||||
|
"start_unit_id": 20,
|
||||||
|
"end_unit_id": 20
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
Each context requires a `context_ref` object and `evidence_refs` and `units`
|
Each source unit has required `id`, `kind`, `text`, and self `ref` fields.
|
||||||
arrays. `context_ref` identifies the first and last included unit. Each
|
`ref` contains `source_id`, `start_unit_id`, and `end_unit_id`, and both unit
|
||||||
evidence entry contains a selected `lane_id` and an original `source_ref`. A
|
endpoints identify that unit's `id`. A unit may also contain source-owned
|
||||||
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
|
`metadata`, an open-ended JSON object. Fixed unit and reference fields are
|
||||||
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
|
strict: consumers must reject unknown fixed fields, malformed units, invalid
|
||||||
unknown fields; unit metadata may contain application-defined JSON values.
|
self-references, units whose `source_id` differs from other units in the same
|
||||||
|
excerpt, and a payload that is not the array described here.
|
||||||
|
|
||||||
## Citations And Context
|
The excerpt preserves each selected unit exactly as represented by the
|
||||||
|
validated generic source document. It does not add evidence-context-specific
|
||||||
|
annotations or reshape source-owned metadata.
|
||||||
|
|
||||||
`evidence_refs` are the authoritative citations. They identify the direct
|
## Selection And Citations
|
||||||
references emitted by accepted normalized artifacts. `context_ref` and the
|
|
||||||
units collection include those cited units plus nearby source units selected by
|
|
||||||
the configured window. They are explanatory context, not widened citations.
|
|
||||||
|
|
||||||
Only accepted outputs from the configured lane allowlist contribute. Rejected,
|
The framework obtains direct source references only through typed evidence
|
||||||
failed, absent, and lane-filtered outputs do not contribute. The artifact never
|
projections of accepted normalized artifacts in the configured lane allowlist.
|
||||||
contains raw input bytes, prompts, model responses, auxiliary reference
|
It validates each reference against the current source document, expands its
|
||||||
content, credentials, or filesystem paths.
|
range by `window_units` source-unit positions on each side, clamps at document
|
||||||
|
boundaries, and takes the union of all expanded ranges. The output contains
|
||||||
|
each selected source unit once in source-document position order, regardless
|
||||||
|
of numeric unit IDs. Repeated references, overlapping windows, and citations
|
||||||
|
from multiple lanes do not duplicate a unit. Rejected, failed, absent,
|
||||||
|
inactive, and unselected lanes contribute nothing.
|
||||||
|
|
||||||
## Ordering And Compatibility
|
Normalized lane artifacts remain authoritative for citations and for which lane
|
||||||
|
cited a range. The excerpt has no lane attribution and must not be used to
|
||||||
|
reconstruct it. Its included nearby units provide reading context only; they
|
||||||
|
do not widen any citation in a lane artifact.
|
||||||
|
|
||||||
The selected lane allowlist is lexical. Contexts and units are in source
|
The excerpt contains at most every generic source unit once. It can therefore
|
||||||
document position order, not numeric unit-ID order. Direct evidence entries
|
equal the complete generic source document when coverage is broad or the
|
||||||
are deterministically ordered by lane and source reference. Overlapping or
|
window is large. No byte-, token-, or compression-size guarantee is made, and
|
||||||
contiguous windows merge, and each source unit appears at most once in the
|
the framework does not truncate the excerpt to meet an arbitrary size limit.
|
||||||
resulting contexts.
|
|
||||||
|
## Consumer Responsibilities And Data Handling
|
||||||
|
|
||||||
The artifact is additive to the JSON bundle and is not a lane payload,
|
The artifact is additive to the JSON bundle and is not a lane payload,
|
||||||
normalized-output count, checkpoint, or generated reference. Consumers that
|
normalized-output count, checkpoint, or generated reference. Consumers that
|
||||||
do not need it must tolerate the absent optional descriptor. Consumers that do
|
do not need it must tolerate an absent descriptor. Consumers that do use it
|
||||||
use it should preserve the artifact and its schema identity with the run
|
should validate the descriptor and payload before use, retain the artifact with
|
||||||
provenance, and should treat its source text and metadata as sensitive durable
|
its schema identity when needed for a run record, and read citations from the
|
||||||
content.
|
corresponding normalized lane artifacts.
|
||||||
|
|
||||||
|
The excerpt contains source-unit text and source-owned metadata and is durable
|
||||||
|
output. Treat it as sensitive source content, apply appropriate access controls
|
||||||
|
and retention, and do not assume its selected form is materially smaller or
|
||||||
|
less sensitive than the original input.
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ root for the logical discovery described here.
|
|||||||
| `warnings.json` | Accepted-output and run warnings. |
|
| `warnings.json` | Accepted-output and run warnings. |
|
||||||
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
||||||
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
||||||
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
|
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
|
||||||
|
|
||||||
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
||||||
accepted only when their media type is `application/json`.
|
accepted only when their media type is `application/json`.
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# PromptKit Integration
|
# PromptKit Integration
|
||||||
|
|
||||||
Notarius pins
|
Notarius pins
|
||||||
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
|
[`gitea.maximumdirect.net/eric/promptkit` v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0)
|
||||||
as its in-process prompt engine. The upstream
|
as its in-process prompt engine. The upstream
|
||||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
|
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/consumers/pkg-promptkit.md)
|
||||||
owns the public engine API, and the upstream
|
owns the public engine API, and the upstream
|
||||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md)
|
||||||
owns prompt, profile, and schema file contracts.
|
owns prompt, profile, and schema file contracts.
|
||||||
|
|
||||||
## Supported Boundary
|
## Supported Boundary
|
||||||
@@ -26,7 +26,7 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
admission exhaustion through `ErrCapacityExceeded`.
|
admission exhaustion through `ErrCapacityExceeded`.
|
||||||
|
|
||||||
The pinned
|
The pinned
|
||||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/backends.go)
|
||||||
owns the registration and backend-capacity contract.
|
owns the registration and backend-capacity contract.
|
||||||
|
|
||||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||||
@@ -52,7 +52,7 @@ Notarius sends one stable effective session through PromptKit's direct session
|
|||||||
field, which is authoritative for provider session behavior. It also retains
|
field, which is authoritative for provider session behavior. It also retains
|
||||||
the same value as the `session_id` prompt variable for maintained prompt
|
the same value as the `session_id` prompt variable for maintained prompt
|
||||||
compatibility. The generated identifier is 76 ASCII characters, within
|
compatibility. The generated identifier is 76 ASCII characters, within
|
||||||
PromptKit v0.5.0's 256-code-point session limit. Session IDs are non-secret
|
PromptKit v0.8.0's 256-code-point session limit. Session IDs are non-secret
|
||||||
correlation identifiers and may be exposed to providers and provider
|
correlation identifiers and may be exposed to providers and provider
|
||||||
observability. The CLI contract owns generation and override behavior.
|
observability. The CLI contract owns generation and override behavior.
|
||||||
|
|
||||||
@@ -84,7 +84,13 @@ configuration and deployment workflow are defined in
|
|||||||
[Configuration](../config.md#promptkit-profiles) and
|
[Configuration](../config.md#promptkit-profiles) and
|
||||||
[Operations](../operations.md#promptkit-profile-deployment).
|
[Operations](../operations.md#promptkit-profile-deployment).
|
||||||
|
|
||||||
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
|
PromptKit owns `base_profile` resolution under its
|
||||||
|
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md).
|
||||||
|
Notarius records the selected leaf identity and resolved target without parsing
|
||||||
|
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||||
|
reach the provider without authorization, which can result in a 401 or 403.
|
||||||
|
|
||||||
|
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
|
||||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||||
@@ -97,6 +103,10 @@ pinned upstream documentation.
|
|||||||
module assets, maps its transport-neutral completion contract, prepares and
|
module assets, maps its transport-neutral completion contract, prepares and
|
||||||
executes requests, validates output, records provenance, captures debug
|
executes requests, validates output, records provenance, captures debug
|
||||||
material, redacts errors, and preserves timeout ownership.
|
material, redacts errors, and preserves timeout ownership.
|
||||||
|
|
||||||
|
PromptKit provider error details do not cross the ordinary completion boundary.
|
||||||
|
Notarius exposes a provider-neutral generation category and optional status;
|
||||||
|
redacted provider details are retained only in requested debug material.
|
||||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||||
@@ -105,4 +115,7 @@ the conventional local backend.
|
|||||||
|
|
||||||
PromptKit API or format changes outside this boundary are not implicitly
|
PromptKit API or format changes outside this boundary are not implicitly
|
||||||
supported. Updating the pinned version requires reviewing the adapter and
|
supported. Updating the pinned version requires reviewing the adapter and
|
||||||
profile/configuration contracts against the upstream documentation.
|
profile/configuration contracts against the upstream documentation. Maintained
|
||||||
|
production prompts use PromptKit's bounded structural-repair contract; their
|
||||||
|
current declaration is one additional repair attempt. Notarius retains the
|
||||||
|
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||||
|
|||||||
@@ -24,10 +24,14 @@ preparation, and runner mechanics after their inputs are supplied.
|
|||||||
|
|
||||||
## Dispatch And Configuration Handoff
|
## Dispatch And Configuration Handoff
|
||||||
|
|
||||||
The root dispatcher handles help, configuration validation, pipeline listing,
|
The root dispatcher handles help, version reporting, configuration validation,
|
||||||
and a pipeline run. It normalizes injectable options before dispatch so that a
|
pipeline listing, and a pipeline run. Version reporting resolves build
|
||||||
missing production dependency fails as a command error rather than reaching
|
information through `internal/buildinfo` before production composition, so the
|
||||||
execution.
|
diagnostic remains available without configuration or runtime collaborators.
|
||||||
|
The public syntax, streams, exit classes, and version semantics are defined by
|
||||||
|
the [CLI reference](../cli.md). Other root commands normalize injectable
|
||||||
|
options before dispatch so that a missing production dependency fails as a
|
||||||
|
command error rather than reaching execution.
|
||||||
|
|
||||||
Commands that need configuration use one shared loader. The CLI discovers the
|
Commands that need configuration use one shared loader. The CLI discovers the
|
||||||
file, parses it through **internal/core/config**, starts from defaults, applies
|
file, parses it through **internal/core/config**, starts from defaults, applies
|
||||||
|
|||||||
@@ -84,14 +84,15 @@ replace it with a complete profile of the same ID from the configured PromptKit
|
|||||||
source. Deployment profile selection is documented in
|
source. Deployment profile selection is documented in
|
||||||
[Configuration](../config.md#promptkit-profiles).
|
[Configuration](../config.md#promptkit-profiles).
|
||||||
|
|
||||||
The transcript assets have distinct consumers. Scene chunking consumes the
|
The D&D transcript assets have distinct consumers. Scene chunking consumes the
|
||||||
complete-session `common-dnd-transcript-full.md`; extraction prompts consume
|
complete-session `common-dnd-transcript-full.md`, while extraction prompts
|
||||||
the current-chunk `common-dnd-transcript-chunk.md`; and NPC, location, and item
|
consume the current-chunk `common-dnd-transcript-chunk.md`. NPC, location, and
|
||||||
normalization consume `common-dnd-transcript-windows.md` alongside their
|
item normalization instead mount the generic semantic-reconciliation
|
||||||
candidate collections. Player, party, glossary, and compatible campaign
|
candidate and transcript-window presentation assets. Player, party, glossary,
|
||||||
references provide disambiguating context only when declared by the active
|
and compatible campaign references provide disambiguating context only when
|
||||||
prompt; they never establish evidence. Reference material is canonically
|
declared by the active prompt; they never establish evidence. Reference
|
||||||
ordered before rendering so equivalent inputs remain stable.
|
material is canonically ordered before rendering so equivalent inputs remain
|
||||||
|
stable.
|
||||||
|
|
||||||
Extraction prompts render the common system and identity messages first, then
|
Extraction prompts render the common system and identity messages first, then
|
||||||
cached campaign references and the cached chunk transcript. Evidence policy and
|
cached campaign references and the cached chunk transcript. Evidence policy and
|
||||||
@@ -101,10 +102,11 @@ reusable extraction prefix identical while preserving the lane-specific suffix.
|
|||||||
|
|
||||||
Scene chunking intentionally uses a different order: system, cached campaign
|
Scene chunking intentionally uses a different order: system, cached campaign
|
||||||
references, uncached module instructions, then the final ephemeral full
|
references, uncached module instructions, then the final ephemeral full
|
||||||
transcript. Entity normalization also has its own order: system, uncached
|
transcript. Entity normalization also has its own order: D&D system, mandatory
|
||||||
module instructions, ephemeral reconciliation policy, uncached candidates, and
|
generic protocol, ephemeral domain semantic instructions, generic candidate
|
||||||
final ephemeral transcript windows. These orders and cache controls are prompt
|
presentation, and final ephemeral generic transcript windows. These orders and
|
||||||
behavior; change them only through the owning manifest and prompt declaration.
|
cache controls are prompt behavior; change them only through the owning
|
||||||
|
manifest and prompt declaration.
|
||||||
|
|
||||||
## Evidence, Candidates, And Normalization
|
## Evidence, Candidates, And Normalization
|
||||||
|
|
||||||
@@ -134,12 +136,46 @@ canonicalize display values and evidence, use source-document order for stable
|
|||||||
output, and issue bounded warnings for changes or collapsed duplicates. NPC,
|
output, and issue bounded warnings for changes or collapsed duplicates. NPC,
|
||||||
item, and location registry normalizers are intentional exceptions: each first
|
item, and location registry normalizers are intentional exceptions: each first
|
||||||
produces a deterministic candidate set, then may use a bounded structured-LLM
|
produces a deterministic candidate set, then may use a bounded structured-LLM
|
||||||
proposal to reconcile identity groups. The proposal selects supplied
|
proposal to reconcile identity groups.
|
||||||
descriptors—names with their candidate source references—not durable IDs.
|
|
||||||
Request-local candidate keys may support resolution internally, but are never
|
## Semantic Registry Reconciliation
|
||||||
included in model input or output. Colliding descriptors are ineligible, and
|
|
||||||
invalid or unusable proposals retain the deterministic result with retry or
|
The three registry normalizers instantiate the domain-neutral
|
||||||
fallback diagnostics; the model does not directly replace durable records.
|
`internal/framework/semanticreconcile` engine with default bounds. Each
|
||||||
|
eligible candidate receives a contiguous, one-based `candidate_id` for that
|
||||||
|
request. The model sees that handle, the candidate label and source-free
|
||||||
|
evidence ranges, plus bounded transcript windows; it returns only duplicate
|
||||||
|
groups of supplied handles and one supplied canonical handle per group. It
|
||||||
|
never returns names, evidence, durable IDs, or replacement records. Identical
|
||||||
|
labels and evidence remain independently selectable because their handles are
|
||||||
|
distinct.
|
||||||
|
|
||||||
|
The generic core owns the mandatory handle protocol, candidate and transcript
|
||||||
|
presentation, the private response schema, source-reference validation,
|
||||||
|
candidate and combined-material limits, structured completion, proposal
|
||||||
|
assessment, stable group ordering, and typed plan-application mechanics. The
|
||||||
|
D&D prompt contributes its system message and registry-specific semantic
|
||||||
|
instructions. The generic registrar registers the shared prompt and schema;
|
||||||
|
the D&D registrar registers each consuming prompt and the fallback profile.
|
||||||
|
|
||||||
|
Fewer than two eligible candidates skips the LLM without a semantic warning.
|
||||||
|
An exceeded bound also skips the call and preserves the deterministic
|
||||||
|
preprocessed registry, adding the registry's bounded fallback warning. Invalid
|
||||||
|
structured output or discarded proposal groups use the normalizer's existing
|
||||||
|
retry contract; retry exhaustion preserves the safe deterministic or
|
||||||
|
partially applied result and emits its bounded fallback warning. Provider,
|
||||||
|
transport, cancellation, and context-material failures remain execution
|
||||||
|
errors.
|
||||||
|
|
||||||
|
Application remains typed and registry-owned. All three policies select the
|
||||||
|
canonical member's normalized display name, union member evidence in source
|
||||||
|
order, preserve ungrouped records, and derive durable identity only after
|
||||||
|
consolidation. NPC IDs derive from the final name. Item IDs also derive from
|
||||||
|
the final name, and a typed guard prevents currency aliases from crossing
|
||||||
|
denominations or mixing currency with non-currency records. Location IDs
|
||||||
|
derive from the final name and final evidence, preserving same-name,
|
||||||
|
parent/child, and distinct physical-place identities. Registry warning scopes,
|
||||||
|
reason codes, and postconditions remain outside the generic core.
|
||||||
|
|
||||||
## Generated References And Grounding
|
## Generated References And Grounding
|
||||||
|
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ backend membership as runtime without performing generation. Fallback assets
|
|||||||
are mounted only when at least one source is registered. The production D&D
|
are mounted only when at least one source is registered. The production D&D
|
||||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||||
profile parsing: an operator-provided matching profile takes precedence over a
|
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||||
fallback profile without Notarius merging either document.
|
fallback profile without Notarius merging either document.
|
||||||
When the registration is absent, a profile selecting `backend: local` fails
|
When the registration is absent, a profile selecting `backend: local` fails
|
||||||
inspection instead of falling back to a built-in or endpoint-only target.
|
inspection instead of falling back to a built-in or endpoint-only target.
|
||||||
@@ -142,12 +142,28 @@ arrangement and its data-only boundary are defined by
|
|||||||
[ADR-0011](../adr/0011-centralize-llm-assets.md), rather than by this runtime
|
[ADR-0011](../adr/0011-centralize-llm-assets.md), rather than by this runtime
|
||||||
guide.
|
guide.
|
||||||
|
|
||||||
|
The generic registrar is the sole production registration owner for the
|
||||||
|
semantic-reconciliation default prompt and private response schema. The
|
||||||
|
domain-neutral reconciliation package also exposes only its mandatory protocol
|
||||||
|
and candidate/transcript presentation files for domain prompt manifests. D&D
|
||||||
|
registry normalizers mount those files while retaining ownership and hashing
|
||||||
|
of their D&D system message, semantic instructions, and complete prompt
|
||||||
|
declaration. The response schema is therefore registered once even though
|
||||||
|
several typed normalizers select it.
|
||||||
|
|
||||||
Mounted prompt assets determine a module's fingerprint. The fingerprint hashes
|
Mounted prompt assets determine a module's fingerprint. The fingerprint hashes
|
||||||
only the module and shared files explicitly selected by its manifest, so an
|
only the module and shared files explicitly selected by its manifest, so an
|
||||||
unrelated asset does not invalidate a checkpoint. Schema loaders validate JSON,
|
unrelated asset does not invalidate a checkpoint. Schema loaders validate JSON,
|
||||||
attach identity and digest metadata, make defensive copies, and expose
|
attach identity and digest metadata, make defensive copies, and expose
|
||||||
diagnostics without raw schema bytes.
|
diagnostics without raw schema bytes.
|
||||||
|
|
||||||
|
Semantic-reconciliation normalizers extend this identity with the shared
|
||||||
|
response-schema digest, framework policy version, and complete limit-policy
|
||||||
|
digest. Their manifest metadata records the same content-free prompt, schema,
|
||||||
|
policy, and limit identities together with domain identity and normalization
|
||||||
|
policies. Request-local handles, source material, proposal content, and raw
|
||||||
|
asset bytes are not checkpoint metadata.
|
||||||
|
|
||||||
Private response schemas validate a model transport envelope. They are not the
|
Private response schemas validate a model transport envelope. They are not the
|
||||||
durable artifact schema and should not be documented as an external wire
|
durable artifact schema and should not be documented as an external wire
|
||||||
contract. Durable formats and compatibility rules remain in the
|
contract. Durable formats and compatibility rules remain in the
|
||||||
@@ -194,14 +210,29 @@ caller context takes precedence. The adapter does not retry capacity failures;
|
|||||||
the pipeline's existing binding attempt policy sees the operational error and
|
the pipeline's existing binding attempt policy sees the operational error and
|
||||||
decides whether to rerun the complete operation.
|
decides whether to rerun the complete operation.
|
||||||
|
|
||||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
PromptKit executes structural repair within its structured-output flow. The
|
||||||
The current production D&D prompt manifests set repair attempts to zero. That
|
maintained production prompt manifests declare one additional repair attempt.
|
||||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
When a resolved binding supplies a repair value, the adapter inspects the
|
||||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
prompt, copies its complete output contract, changes only the repair limit, and
|
||||||
rejection is a recorded output rather than a provider error. The pipeline owns
|
passes that complete replacement contract to PromptKit. This preserves the
|
||||||
attempt lifecycle, validation chains, and retry diagnostics; see
|
prompt's output format, validation mode, schema, and provider structured-output
|
||||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
settings.
|
||||||
[binding reference](../config.md#module-bindings-and-validators).
|
|
||||||
|
A successful repair is an ordinary successful completion, not a warning. The
|
||||||
|
adapter reports PromptKit's actual repair count and its cumulative usage
|
||||||
|
directly, without adding the initial and corrective counts again. Debug prompt
|
||||||
|
material records the configured complete contract; debug response material
|
||||||
|
records the repaired response and actual validation result. If the repair
|
||||||
|
budget is exhausted, the adapter retains the final raw bytes and debug material
|
||||||
|
and reports `ErrInvalidStructuredOutput`. Generation failures during an initial
|
||||||
|
or corrective call remain provider-neutral operational errors with the same
|
||||||
|
redaction boundary.
|
||||||
|
|
||||||
|
Structural repair does not replace pipeline retry behavior: a binding's
|
||||||
|
configured retry count reruns its complete stage attempt after an error or
|
||||||
|
rejection. The pipeline owns attempt lifecycle, validation chains, and retry
|
||||||
|
diagnostics; see [Pipeline Internals](pipeline.md#validation-retries-and-output)
|
||||||
|
and the [binding reference](../config.md#module-bindings-and-validators).
|
||||||
|
|
||||||
## Timeout Ownership
|
## Timeout Ownership
|
||||||
|
|
||||||
@@ -240,6 +271,9 @@ redacted before it crosses the runtime boundary. Known-secret redaction is
|
|||||||
available to other runtime collaborators; it does not make prompt or response
|
available to other runtime collaborators; it does not make prompt or response
|
||||||
contents safe for general logging.
|
contents safe for general logging.
|
||||||
|
|
||||||
|
Generation failures expose an application-owned category and optional HTTP
|
||||||
|
status. Provider code, type, and message remain debug-only, after redaction.
|
||||||
|
|
||||||
## Failure Boundaries
|
## Failure Boundaries
|
||||||
|
|
||||||
- Construction fails for missing asset registries, mutually exclusive profile
|
- Construction fails for missing asset registries, mutually exclusive profile
|
||||||
|
|||||||
@@ -42,14 +42,23 @@ generic source references and must use the codec's exact Go type. It does not
|
|||||||
interpret surrounding context or publish files; the pipeline validates the
|
interpret surrounding context or publish files; the pipeline validates the
|
||||||
capability during preparation and the output boundary owns publication. See
|
capability during preparation and the output boundary owns publication. See
|
||||||
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
||||||
for the durable result.
|
for the durable source-unit excerpt. Lane artifacts retain citation and lane
|
||||||
|
provenance; the framework does not add either to that published excerpt.
|
||||||
|
|
||||||
|
An artifact family is broader than a module: it owns the cohesive domain
|
||||||
|
feature across its artifact type, codec, stage modules, validators, prompt
|
||||||
|
policy, schemas, identity helpers, and reference projections. An extractor and
|
||||||
|
normalizer in one artifact family remain independently registered modules in
|
||||||
|
their respective pipeline stages. This ownership vocabulary does not create a
|
||||||
|
new registry or change the fixed pipeline.
|
||||||
|
|
||||||
## Production Composition
|
## Production Composition
|
||||||
|
|
||||||
Production composition is intentionally split by family:
|
Production composition is intentionally split by family:
|
||||||
|
|
||||||
- The generic registrar provides the unit chunker, generic JSON validators,
|
- The generic registrar provides the unit chunker, generic JSON validators,
|
||||||
and JSON output encoder.
|
JSON output encoder, and shared semantic-reconciliation prompt and response
|
||||||
|
schema assets.
|
||||||
- The Seriatim registrar provides the transcript input adapter. Its external
|
- The Seriatim registrar provides the transcript input adapter. Its external
|
||||||
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
|
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
|
||||||
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
|
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
|
||||||
@@ -60,6 +69,36 @@ The CLI owns the composition that invokes these registrars. A module package
|
|||||||
may register its own family but must not assemble the CLI or make framework
|
may register its own family but must not assemble the CLI or make framework
|
||||||
packages depend on production extensions.
|
packages depend on production extensions.
|
||||||
|
|
||||||
|
## Semantic Reconciliation
|
||||||
|
|
||||||
|
`internal/framework/semanticreconcile` is a domain-neutral strategy used by a
|
||||||
|
typed normalize module; it is not itself a selectable stage module. A
|
||||||
|
source-backed artifact-family normalizer projects its deterministic records
|
||||||
|
into contextual candidates and owned typed record envelopes, supplies its
|
||||||
|
chosen prompt identity and resolved LLM profile, and constructs an engine with
|
||||||
|
explicit limits. The core filters invalid evidence, assigns contiguous
|
||||||
|
request-local integer handles, renders bounded candidate and transcript
|
||||||
|
materials, invokes the structured-completion boundary, and assesses the
|
||||||
|
returned duplicate groups into a stable non-overlapping plan.
|
||||||
|
|
||||||
|
The normalizer then applies that plan through a typed `ApplicationPolicy`. The
|
||||||
|
core preserves ungrouped records, contribution order, and provenance while the
|
||||||
|
artifact family owns group guards, field and evidence consolidation, durable
|
||||||
|
ID derivation, retry and fallback presentation, warnings, and postconditions.
|
||||||
|
Request-local handles do not enter the typed value or durable artifact. Fewer
|
||||||
|
than two eligible candidates skips model invocation; exceeding a candidate or
|
||||||
|
combined-material bound preserves the deterministic result under the family's
|
||||||
|
fallback policy. Provider, transport, cancellation, and context-construction
|
||||||
|
failures remain execution errors.
|
||||||
|
|
||||||
|
The core supplies a conservative generic prompt and the single private
|
||||||
|
response schema. A domain prompt may substitute its semantic instructions but
|
||||||
|
mounts the core-owned protocol and candidate/transcript presentation assets.
|
||||||
|
Prompt, schema, policy, and limit identities participate in manifest metadata
|
||||||
|
and checkpoint fingerprints. The generic registrar owns production
|
||||||
|
registration of those shared assets; a consuming domain registrar owns only
|
||||||
|
its domain prompt.
|
||||||
|
|
||||||
## Adding Or Changing A Module
|
## Adding Or Changing A Module
|
||||||
|
|
||||||
1. Choose the pipeline stage and the typed artifact boundary. Put external
|
1. Choose the pipeline stage and the typed artifact boundary. Put external
|
||||||
|
|||||||
@@ -25,10 +25,12 @@ physical state roots.
|
|||||||
| Area | Implemented owners | Responsibility |
|
| Area | Implemented owners | Responsibility |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
||||||
|
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
|
||||||
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
||||||
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
||||||
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
||||||
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
|
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
|
||||||
|
| Semantic reconciliation | **internal/framework/semanticreconcile** | Bounded source-backed candidate preparation, request-local handle proposals, deterministic assessment, typed plan application, and reconciliation identity metadata; see [Module Internals](modules.md#semantic-reconciliation) and [D&D Module Internals](dnd.md#semantic-registry-reconciliation). |
|
||||||
| Embedded LLM content | **assets** | Read-only centralized LLM-facing content, scoped by its consuming package; see [LLM Runtime](llm.md#prompt-and-schema-assets) and [D&D Module Internals](dnd.md#prompt-construction). |
|
| Embedded LLM content | **assets** | Read-only centralized LLM-facing content, scoped by its consuming package; see [LLM Runtime](llm.md#prompt-and-schema-assets) and [D&D Module Internals](dnd.md#prompt-construction). |
|
||||||
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
|
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
|
||||||
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
|
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
|
||||||
@@ -49,8 +51,9 @@ the CLI composition boundary.
|
|||||||
composition, and path safety.
|
composition, and path safety.
|
||||||
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
|
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
|
||||||
profiles, and secret handling.
|
profiles, and secret handling.
|
||||||
- [Module Internals](modules.md): generic extension registration, module
|
- [Module Internals](modules.md): generic extension registration, artifact
|
||||||
construction, validation, and reference mechanics.
|
families, module construction, semantic reconciliation, validation, and
|
||||||
|
reference mechanics.
|
||||||
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
|
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
|
||||||
reference projections, and lane-specific exceptions. Durable D&D and
|
reference projections, and lane-specific exceptions. Durable D&D and
|
||||||
Seriatim data shapes remain in the [integration contracts](../integrations/).
|
Seriatim data shapes remain in the [integration contracts](../integrations/).
|
||||||
|
|||||||
@@ -36,9 +36,16 @@ assigns a deterministic resolved-composition digest. The resolved pipeline
|
|||||||
contains bindings and declared reference targets, not external reference bytes.
|
contains bindings and declared reference targets, not external reference bytes.
|
||||||
After selection, the resolver applies command, binding, and pipeline profile
|
After selection, the resolver applies command, binding, and pipeline profile
|
||||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||||
an empty resolved binding profile. Deterministic bindings remain profile-free.
|
an empty resolved binding profile. It resolves structural output repair
|
||||||
These effective values are part of the digest, so execution and checkpoint
|
separately: a binding's `structured_output_repair_attempts` value wins, then a
|
||||||
consumers do not repeat profile inheritance.
|
pipeline value applies to LLM-backed bindings and validators, and omission
|
||||||
|
leaves the prompt-owned policy intact. An explicit repair value on a
|
||||||
|
deterministic binding is rejected. Resolved bindings own copied repair values,
|
||||||
|
and these effective values are part of the digest, so execution and checkpoint
|
||||||
|
consumers do not repeat profile inheritance or configuration resolution.
|
||||||
|
Each LLM request receives its own copy of that resolved value. PromptKit spends
|
||||||
|
it only for structural correction inside one completion; the runner's binding
|
||||||
|
retry policy remains the separate outer budget for complete stage attempts.
|
||||||
Configuration resolution supplies the selected profile and catalog; see
|
Configuration resolution supplies the selected profile and catalog; see
|
||||||
[Configuration Internals](configuration.md).
|
[Configuration Internals](configuration.md).
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,23 @@ This is the canonical guide for operating Notarius runtime state. The
|
|||||||
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
||||||
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
||||||
|
|
||||||
|
## Source Deployment
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. Install a pinned source release
|
||||||
|
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Pin the exact tag in deployment automation rather than following a branch.
|
||||||
|
Use [`notarius --version`](cli.md#command-summary) as a diagnostic after
|
||||||
|
installation; its syntax and semantics are owned by the [CLI reference](cli.md).
|
||||||
|
The maintainer publication process, including tag guards and verification,
|
||||||
|
belongs to [Source Releases](release.md). macOS builds are best-effort for
|
||||||
|
development, and Windows is unsupported.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
Each run can use independent roots with different retention and access-control
|
Each run can use independent roots with different retention and access-control
|
||||||
@@ -72,6 +89,9 @@ provider call or credentials:
|
|||||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
|
An unset optional `api_key_env` reaches the provider without authorization and
|
||||||
|
may receive a 401 or 403 response.
|
||||||
|
|
||||||
Profile paths are currently resolved from the process working directory, not
|
Profile paths are currently resolved from the process working directory, not
|
||||||
from the configuration file. The complete example's
|
from the configuration file. The complete example's
|
||||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||||
@@ -110,9 +130,9 @@ are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
|
|||||||
[evidence context](integrations/evidence-context.md) contains source-unit text
|
[evidence context](integrations/evidence-context.md) contains source-unit text
|
||||||
and metadata. It is not a cache or debug artifact: retain it with the output
|
and metadata. It is not a cache or debug artifact: retain it with the output
|
||||||
bundle only for as long as consumers need it, and apply source-content access
|
bundle only for as long as consumers need it, and apply source-content access
|
||||||
controls to the entire bundle. Selected lanes may collectively cite most of a
|
controls to the entire bundle. Its selected source-unit excerpt may include
|
||||||
transcript, so a broad allowlist can make the evidence artifact nearly as
|
every source unit once when coverage is broad or its configured window is
|
||||||
sensitive and large as the source itself.
|
large, so do not assume a byte or token reduction or reduced sensitivity.
|
||||||
|
|
||||||
## Chunk-Plan Cache
|
## Chunk-Plan Cache
|
||||||
|
|
||||||
@@ -255,14 +275,29 @@ Provider execution settings and the generation timeout come from the selected
|
|||||||
PromptKit profile. The invocation-only **--reasoning-effort** and
|
PromptKit profile. The invocation-only **--reasoning-effort** and
|
||||||
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
||||||
for all LLM-backed calls in one run without changing the profile. PromptKit
|
for all LLM-backed calls in one run without changing the profile. PromptKit
|
||||||
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
|
structural output repair happens within one structured-completion call. Its
|
||||||
complete module operation and validation chain as defined by
|
effective `structured_output_repair_attempts` limit is resolved from the
|
||||||
[module bindings](config.md#module-bindings-and-validators).
|
selected binding, then the pipeline, then the prompt declaration; see
|
||||||
|
[module bindings](config.md#module-bindings-and-validators). This is distinct
|
||||||
|
from Notarius binding **retries**, which rerun the complete module operation
|
||||||
|
and validation chain and do not consume or replenish the structural-repair
|
||||||
|
limit. The maintained production prompts declare one repair attempt, paid only
|
||||||
|
after a structural failure. One structured completion with repair budget **R**
|
||||||
|
makes at most **R + 1** serial provider calls. If one stage attempt performs
|
||||||
|
**C** structured completions, a binding with **retries: N** has a maximum of
|
||||||
|
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
|
||||||
|
corresponding invocation counts and budgets. This is an upper bound, not a
|
||||||
|
promise that every call reaches a provider.
|
||||||
|
|
||||||
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
||||||
effective generation timeout adds an inner request deadline, while zero
|
effective generation timeout adds an inner request deadline, while zero
|
||||||
disables only that generation deadline. The HTTP client timeout remains a
|
disables only that generation deadline. The HTTP client timeout remains a
|
||||||
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
||||||
|
Repairs are serial within the same caller context, so their worst-case latency
|
||||||
|
and cost follow the provider-call bound above; provision run deadlines and
|
||||||
|
provider budgets accordingly. Credentials remain optional unless the selected
|
||||||
|
PromptKit profile requires one, in which case preparation fails before a
|
||||||
|
provider call when its configured credential is unavailable.
|
||||||
The pinned upstream boundary and profile-format links are in
|
The pinned upstream boundary and profile-format links are in
|
||||||
[PromptKit Integration](integrations/pkg-promptkit.md).
|
[PromptKit Integration](integrations/pkg-promptkit.md).
|
||||||
|
|
||||||
@@ -279,6 +314,11 @@ positive value makes the effective active local-generation bound the smaller
|
|||||||
of **total_llm** and that local limit, so a local limit of four permits no more
|
of **total_llm** and that local limit, so a local limit of four permits no more
|
||||||
than four active local generations.
|
than four active local generations.
|
||||||
|
|
||||||
|
The Notarius scheduler admits one logical structured completion and holds that
|
||||||
|
permit while PromptKit performs its serial corrective calls. PromptKit applies
|
||||||
|
its selected-backend admission to each provider call; Notarius does not
|
||||||
|
reacquire a permit or add another scheduler for a repair.
|
||||||
|
|
||||||
For a positive local limit, PromptKit owns its default waiting capacity and
|
For a positive local limit, PromptKit owns its default waiting capacity and
|
||||||
admission behavior. When a PromptKit backend has admitted all active and queued
|
admission behavior. When a PromptKit backend has admitted all active and queued
|
||||||
work, a new call fails as capacity exhaustion before generation. The adapter
|
work, a new call fails as capacity exhaustion before generation. The adapter
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ DAGs or a general workflow language. Every stage remains explicit; general
|
|||||||
chunking, merging, or normalization behavior must not be hidden inside an
|
chunking, merging, or normalization behavior must not be hidden inside an
|
||||||
extractor.
|
extractor.
|
||||||
|
|
||||||
|
A stage module is one configured implementation of one pipeline stage. An
|
||||||
|
artifact family is the cohesive domain feature that owns an artifact across
|
||||||
|
the explicit stages and supporting codecs, validators, prompts, identity
|
||||||
|
rules, and reference projections. Artifact-family ownership does not combine
|
||||||
|
stages or alter the fixed pipeline.
|
||||||
|
|
||||||
Input and chunking are pipeline-wide. Each selected artifact lane owns its
|
Input and chunking are pipeline-wide. Each selected artifact lane owns its
|
||||||
extract, merge, and normalize stages, and the output stage aggregates the run's
|
extract, merge, and normalize stages, and the output stage aggregates the run's
|
||||||
lane outcomes.
|
lane outcomes.
|
||||||
@@ -39,6 +45,12 @@ implementations. Domain-neutral model and framework layers provide reusable
|
|||||||
policy, contracts, and orchestration. Concrete input, pipeline, output, and
|
policy, contracts, and orchestration. Concrete input, pipeline, output, and
|
||||||
validation extensions depend inward on those generic layers.
|
validation extensions depend inward on those generic layers.
|
||||||
|
|
||||||
|
Semantic reconciliation is one such domain-neutral framework mechanism. It
|
||||||
|
prepares bounded source context, invokes a shared model-judgment protocol,
|
||||||
|
validates proposals, and applies safe plans through typed policies supplied by
|
||||||
|
the consuming artifact family. It does not own domain identity, durable IDs,
|
||||||
|
warning semantics, or artifact construction rules.
|
||||||
|
|
||||||
Generic layers must not depend on production extensions. Concrete extensions
|
Generic layers must not depend on production extensions. Concrete extensions
|
||||||
must not compose the application or take ownership of process behavior. The
|
must not compose the application or take ownership of process behavior. The
|
||||||
current packages implementing these layers are inventoried in
|
current packages implementing these layers are inventoried in
|
||||||
@@ -183,11 +195,20 @@ The caller of the LLM owns prompt selection, prompt inputs, response schema,
|
|||||||
and interpretation of structured output. Provider adapters do not own source-
|
and interpretation of structured output. Provider adapters do not own source-
|
||||||
or domain-specific prompt logic.
|
or domain-specific prompt logic.
|
||||||
|
|
||||||
|
PromptKit owns bounded structural correction within one structured completion.
|
||||||
|
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
|
||||||
|
the two budgets must remain separate.
|
||||||
|
|
||||||
When a model selects an application entity, callers must supply a contextual
|
When a model selects an application entity, callers must supply a contextual
|
||||||
selection and deterministically attach the opaque application identity whenever
|
selection and deterministically attach the opaque application identity whenever
|
||||||
the selection resolves exactly. Models do not receive or reproduce opaque
|
the selection resolves exactly. Models do not receive or reproduce opaque
|
||||||
application identifiers; [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
|
application identifiers. Semantic reconciliation may instead expose
|
||||||
records the rationale and limited request-local-label exception.
|
contiguous, one-based candidate handles that exist only for one request;
|
||||||
|
deterministic code resolves them before typed application, and they never
|
||||||
|
become durable identity. This is the approved request-local-label application
|
||||||
|
of [ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md)
|
||||||
|
recorded by
|
||||||
|
[ADR-0013](../adr/0013-use-request-local-candidate-handles-for-semantic-reconciliation.md).
|
||||||
|
|
||||||
LLM calls and other external operations accept cancellation and respect
|
LLM calls and other external operations accept cancellation and respect
|
||||||
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
|
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
|
||||||
@@ -244,6 +265,16 @@ contain application data and therefore inherits its sensitivity; operators own
|
|||||||
access controls and retention. Physical layout and operation are defined in
|
access controls and retention. Physical layout and operation are defined in
|
||||||
[Operations](../operations.md).
|
[Operations](../operations.md).
|
||||||
|
|
||||||
|
## Platform And Distribution
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. macOS is supported only as a
|
||||||
|
best-effort development and compilation environment, while Windows is
|
||||||
|
unsupported. Notarius distributes source releases only: an immutable source
|
||||||
|
tag and its checked-in release note identify a release. The project does not
|
||||||
|
publish executable binaries, archives, installers, container images,
|
||||||
|
checksums, signatures, or package-manager entries. Maintainer release commands
|
||||||
|
and tag guards belong to [Source Releases](../release.md).
|
||||||
|
|
||||||
## Architectural Non-Goals
|
## Architectural Non-Goals
|
||||||
|
|
||||||
Notarius does not aim to provide:
|
Notarius does not aim to provide:
|
||||||
|
|||||||
@@ -65,6 +65,8 @@ secret values.
|
|||||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
|
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
|
||||||
|
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
@@ -95,6 +97,14 @@ runtime state and how to operate or recover the application. When a workflow
|
|||||||
crosses these topics, choose the document that owns the task and link to the
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
other contracts.
|
other contracts.
|
||||||
|
|
||||||
|
### Releases
|
||||||
|
|
||||||
|
`docs/release.md` owns the source-release procedure. Release notes are
|
||||||
|
historical summaries, not current-state contract owners: the checked-in note at
|
||||||
|
an immutable tag records that release, while current canonical documentation
|
||||||
|
must change with the behavior it describes. Do not use a release note to defer
|
||||||
|
or replace current documentation updates.
|
||||||
|
|
||||||
### Contracts And Implementation
|
### Contracts And Implementation
|
||||||
|
|
||||||
Integration and API documents define externally observable shapes and
|
Integration and API documents define externally observable shapes and
|
||||||
|
|||||||
164
docs/release.md
Normal file
164
docs/release.md
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
# Source Releases
|
||||||
|
|
||||||
|
This procedure is for maintainers publishing Notarius source releases. A
|
||||||
|
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
|
||||||
|
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
|
||||||
|
candidate after publication; it does not publish or repair a release.
|
||||||
|
|
||||||
|
Notarius publishes no binaries, archives, checksums, signatures, containers,
|
||||||
|
package-manager entries, or Gitea release objects. Windows is not supported.
|
||||||
|
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
|
||||||
|
`v0.3.0` tags.
|
||||||
|
|
||||||
|
## Select And Describe The Release
|
||||||
|
|
||||||
|
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
|
||||||
|
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
|
||||||
|
documented CLI, configuration, durable artifact, integration, or operating
|
||||||
|
contract when its note explains the impact and required operator action. A
|
||||||
|
patch release must not intentionally break those documented contracts within
|
||||||
|
its minor line.
|
||||||
|
|
||||||
|
Create the version-matched note as part of the candidate. Every new note uses
|
||||||
|
this structure, with concise, truthful content in each section:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# Notarius vMAJOR.MINOR.PATCH
|
||||||
|
|
||||||
|
This release ...
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
```
|
||||||
|
|
||||||
|
The note is a historical summary. Link to current canonical documentation for
|
||||||
|
exact behavior, and update that documentation in the candidate rather than
|
||||||
|
using the note as a substitute.
|
||||||
|
|
||||||
|
## Prepare The Candidate
|
||||||
|
|
||||||
|
Set the selected release version and disable Go workspace use for every
|
||||||
|
candidate command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_VERSION=vMAJOR.MINOR.PATCH
|
||||||
|
export RELEASE_VERSION GOWORK=off
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the shared source-candidate checks from the repository. They cover module
|
||||||
|
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
|
||||||
|
configuration validation, and the Linux and Darwin command-build matrix:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./scripts/check-release-source.sh "$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Before committing, manually follow every changed local Markdown link and
|
||||||
|
review the candidate for unintended files, generated output, credentials, or
|
||||||
|
other unrelated changes. Commit the release note and all affected current
|
||||||
|
documentation, then run the shared checker against that exact candidate. Push
|
||||||
|
the candidate commit to `main` only after it succeeds. Record the exact commit
|
||||||
|
only after that push:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
For private-module installation, configure standard `GOPRIVATE` matching this
|
||||||
|
module and ordinary Git authentication for the hosting service before running
|
||||||
|
the verification below. The exact authentication mechanism belongs to the
|
||||||
|
maintainer environment; never record credentials or environment dumps in a
|
||||||
|
release note, command history, or repository file.
|
||||||
|
|
||||||
|
## Guard And Publish The Tag
|
||||||
|
|
||||||
|
Fetch current remote references, then run this guard without editing the
|
||||||
|
candidate. It requires `main`, a clean worktree and index, disabled workspace
|
||||||
|
use, a stable release version, the recorded and pushed commit, a matching note,
|
||||||
|
and unused local and remote tags:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git fetch origin main --tags
|
||||||
|
|
||||||
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
test "$GOWORK" = off
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
|
||||||
|
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
|
||||||
|
test -s "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
|
||||||
|
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
|
||||||
|
done
|
||||||
|
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Create an explicitly lightweight tag against the guarded commit, verify its
|
||||||
|
target, and push only that tag ref:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
|
||||||
|
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
|
||||||
|
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
|
||||||
|
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
Never use `git push --tags`, move a published tag, or delete a published tag.
|
||||||
|
|
||||||
|
## Verify The Published Release
|
||||||
|
|
||||||
|
Confirm that the remote tag still points at the guarded commit and that the
|
||||||
|
note is available from the tagged tree:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
|
||||||
|
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
|
||||||
|
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify a fresh source installation and its diagnostic version. The temporary
|
||||||
|
directory confines the installed command to this check:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_verification_dir=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
|
||||||
|
mkdir -p "$release_verification_dir/bin"
|
||||||
|
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
|
||||||
|
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
|
||||||
|
equivalent source verification when local installation policy requires it.
|
||||||
|
`notarius --version` is diagnostic only; downstream compatibility remains
|
||||||
|
defined by the published receipt and artifact contracts.
|
||||||
|
|
||||||
|
## Failure And Correction Policy
|
||||||
|
|
||||||
|
If candidate validation fails before publication, fix the candidate on `main`,
|
||||||
|
rerun the shared checker, and repeat the guards. An unpublished local tag may
|
||||||
|
be deleted after inspection.
|
||||||
|
|
||||||
|
If the remote tag or tag CI reveals a defect, leave the published tag intact.
|
||||||
|
Fix the defect on `main`, choose a new patch version, write a new matching
|
||||||
|
note, and repeat this procedure. Do not weaken tag immutability or add release
|
||||||
|
assets as a workaround.
|
||||||
@@ -1,516 +0,0 @@
|
|||||||
# Codebase Audit Plan
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document defines a repository-wide audit of Notarius for correctness,
|
|
||||||
efficiency, maintainability, and clarity. The audit should identify concrete
|
|
||||||
improvements without treating abstraction, fewer lines, or higher test coverage
|
|
||||||
as goals in themselves.
|
|
||||||
|
|
||||||
The audit is intentionally separate from implementation. Its findings should
|
|
||||||
be evidence-backed and sufficiently specific to support a later remediation
|
|
||||||
roadmap, but the audit should not modify production code, tests, assets, or
|
|
||||||
current-behavior documentation.
|
|
||||||
|
|
||||||
## Governing Principles
|
|
||||||
|
|
||||||
The audit must preserve the architecture and testing policies in
|
|
||||||
`docs/policy/architecture.md` and `docs/policy/testing.md`.
|
|
||||||
|
|
||||||
In particular:
|
|
||||||
|
|
||||||
- Notarius remains a fixed, staged pipeline rather than a general workflow
|
|
||||||
engine.
|
|
||||||
- Generic framework packages must remain domain-neutral, and production
|
|
||||||
modules must not acquire CLI or physical-state responsibilities.
|
|
||||||
- Typed artifact boundaries, exact codec compatibility, deterministic ordering,
|
|
||||||
whole-output validation, and generated-reference provenance are correctness
|
|
||||||
properties, not incidental complexity to be optimized away.
|
|
||||||
- The root `assets` package remains a content-only dependency leaf.
|
|
||||||
- Shared helpers should protect demonstrated common semantics. Similar-looking
|
|
||||||
code with different ownership, error policy, identity rules, or type contracts
|
|
||||||
should remain separate.
|
|
||||||
- Tests should protect durable behavior and meaningful risks. The audit should
|
|
||||||
not recommend tests merely to increase coverage or freeze implementation
|
|
||||||
details.
|
|
||||||
- Efficiency claims must distinguish measured or structurally credible costs
|
|
||||||
from cosmetic line-count reductions. Optimizing local CPU work that is
|
|
||||||
insignificant beside an LLM call is low priority unless it also simplifies
|
|
||||||
correctness or applies to large inputs.
|
|
||||||
|
|
||||||
## Audit Questions
|
|
||||||
|
|
||||||
Every audited area should be examined through the following questions.
|
|
||||||
|
|
||||||
### Correctness
|
|
||||||
|
|
||||||
- Are documented architecture invariants enforced at the correct boundary?
|
|
||||||
- Can invalid configuration, incompatible artifact types, malformed references,
|
|
||||||
or unavailable dependencies reach execution when they could be rejected
|
|
||||||
during resolution or preparation?
|
|
||||||
- Are nil, empty, absent, rejected, failed, and canceled states distinguished
|
|
||||||
consistently?
|
|
||||||
- Are stored or returned slices, maps, byte slices, options, metadata, source
|
|
||||||
documents, references, and artifacts defensively owned where required?
|
|
||||||
- Are public ordering, selected errors, warnings, and checkpoint decisions
|
|
||||||
deterministic regardless of map or goroutine completion order?
|
|
||||||
- Do cancellation, retry, validation, and partial-work semantics match their
|
|
||||||
documented ownership?
|
|
||||||
- Do checkpoint and chunk-plan identities include every semantic dependency and
|
|
||||||
exclude scheduling-only or diagnostic state?
|
|
||||||
- Can auxiliary references accidentally become source evidence, or can
|
|
||||||
generated references bypass codec, schema, provenance, or step-order checks?
|
|
||||||
- Can provider-specific values, credentials, or source content escape through
|
|
||||||
errors, manifests, debug summaries, cache state, or logs?
|
|
||||||
- Do schemas, codecs, candidate decoders, normalizers, and validators agree on
|
|
||||||
the exact durable contract without silently accepting incompatible shapes?
|
|
||||||
|
|
||||||
### Duplication And Shared Mechanics
|
|
||||||
|
|
||||||
- Which exact or near-duplicate implementations express the same invariant and
|
|
||||||
failure policy?
|
|
||||||
- Has duplicated code already drifted in naming, nil handling, canonicalization,
|
|
||||||
metadata, fingerprints, validation, or diagnostics?
|
|
||||||
- Would a helper have a natural owner and a smaller, clearer contract than the
|
|
||||||
duplicated callers?
|
|
||||||
- Can an extraction preserve static typing and package ownership, or would it
|
|
||||||
require reflection, `any`, callbacks with many policy parameters, or a
|
|
||||||
domain-neutral package importing domain concepts?
|
|
||||||
- Is repeated code required by a small interface adapter or typed registration
|
|
||||||
boundary and therefore clearer when left explicit?
|
|
||||||
|
|
||||||
As a default heuristic, prioritize a shared helper when identical semantics
|
|
||||||
appear in three or more production sites, or in two sites where divergence
|
|
||||||
would create a meaningful correctness risk. Do not use that heuristic as a
|
|
||||||
quota: one substantial duplicate may warrant extraction, while widespread
|
|
||||||
one-line interface methods may not.
|
|
||||||
|
|
||||||
### Simplicity And Idiomatic Go
|
|
||||||
|
|
||||||
- Does a function combine orchestration, policy, transformation, persistence,
|
|
||||||
and reporting that could be separated along existing ownership boundaries?
|
|
||||||
- Are repeated scans, sorts, conversions, clones, encodes, or decodes doing work
|
|
||||||
that can safely occur once?
|
|
||||||
- Are intermediate representations necessary, or can a value be validated,
|
|
||||||
canonicalized, and mapped in one comprehensible pass?
|
|
||||||
- Are maps, sets, stable sorts, generics, standard-library helpers, and error
|
|
||||||
wrapping used idiomatically?
|
|
||||||
- Are abstractions earning their complexity, or are interfaces, option layers,
|
|
||||||
wrappers, aliases, compatibility paths, and private types left over after a
|
|
||||||
completed migration?
|
|
||||||
- Are there unreachable error branches, redundant fingerprints or digests,
|
|
||||||
duplicated sources of truth, or accessors used only by tests?
|
|
||||||
- Can a smaller implementation preserve exact observable behavior and safety
|
|
||||||
properties?
|
|
||||||
|
|
||||||
### Explanatory Comments
|
|
||||||
|
|
||||||
Comments should be recommended where the code is necessarily complex because
|
|
||||||
it preserves a non-obvious invariant. Good candidates include:
|
|
||||||
|
|
||||||
- concurrency coordination, cancellation, and stable error selection;
|
|
||||||
- checkpoint identity, reuse, forced recomputation, and dependency invalidation;
|
|
||||||
- typed erasure and restoration at framework boundaries;
|
|
||||||
- generated-reference ordering and provenance;
|
|
||||||
- canonicalization and identity resolution where registry evidence differs
|
|
||||||
from occurrence evidence;
|
|
||||||
- prompt ordering or input identity required for backend caching; and
|
|
||||||
- path confinement, atomic publication, redaction, or terminal error precedence.
|
|
||||||
|
|
||||||
Recommend comments that explain *why* a step or ordering constraint exists and
|
|
||||||
what would break if it changed. Do not recommend comments that narrate syntax,
|
|
||||||
repeat a function name, duplicate current-behavior documentation, or preserve
|
|
||||||
implementation history.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
- Is each consequential invariant protected at the narrowest stable boundary?
|
|
||||||
- Are concurrency, cancellation, retries, recovery, compatibility, path safety,
|
|
||||||
and data-integrity behavior credibly exercised?
|
|
||||||
- Do higher-level contract tests duplicate lower-level cases without adding
|
|
||||||
integration confidence?
|
|
||||||
- Are tests coupled to private constants, helper shape, exact prose, full error
|
|
||||||
strings, or collaborator choreography rather than behavior?
|
|
||||||
- Can repetitive fixtures or fakes be simplified without creating a test
|
|
||||||
framework more complex than the tests?
|
|
||||||
- Would a focused fuzz test, race test, or package-level invariant test protect
|
|
||||||
a realistic risk better than several example tests?
|
|
||||||
|
|
||||||
## Evidence And Finding Standards
|
|
||||||
|
|
||||||
Static metrics and textual similarity are discovery aids, not findings. A long
|
|
||||||
function may be a clear linear coordinator; identical methods may be useful
|
|
||||||
typed adapters. Every reported finding must include:
|
|
||||||
|
|
||||||
1. a concise title and severity;
|
|
||||||
2. exact files and symbols;
|
|
||||||
3. the observed behavior or structural evidence;
|
|
||||||
4. the correctness, efficiency, maintenance, or comprehension impact;
|
|
||||||
5. a concrete recommended direction;
|
|
||||||
6. important invariants the remediation must preserve;
|
|
||||||
7. focused validation that would demonstrate success; and
|
|
||||||
8. whether the recommendation is independent or should be grouped with another
|
|
||||||
finding.
|
|
||||||
|
|
||||||
Use these severities:
|
|
||||||
|
|
||||||
- **High:** a credible risk of corrupt output, unsafe state handling, secret
|
|
||||||
exposure, stale reuse, deadlock, nondeterminism, or violated external
|
|
||||||
contract.
|
|
||||||
- **Medium:** a plausible behavioral defect, meaningful wasted work on common
|
|
||||||
paths, or complexity/duplication likely to cause future correctness drift.
|
|
||||||
- **Low:** a contained simplification, small efficiency improvement, dead code,
|
|
||||||
naming issue, or missing explanation with no current behavioral failure.
|
|
||||||
|
|
||||||
The audit should explicitly record examined areas with no findings. This makes
|
|
||||||
coverage visible and prevents later agents from repeatedly rediscovering the
|
|
||||||
same safe design.
|
|
||||||
|
|
||||||
## Repository Areas
|
|
||||||
|
|
||||||
### 1. Architecture And Dependency Boundaries
|
|
||||||
|
|
||||||
Inspect `docs/policy/architecture.md`, `docs/adr/`, `docs/internal/overview.md`,
|
|
||||||
package imports, module registrars, and the CLI composition root.
|
|
||||||
|
|
||||||
Look for:
|
|
||||||
|
|
||||||
- framework or core code depending on production modules;
|
|
||||||
- modules depending on CLI, physical roots, or provider-specific types;
|
|
||||||
- domain knowledge placed in generic helpers;
|
|
||||||
- duplicated registries or composition policy outside the owning registrar;
|
|
||||||
- abstractions that turn the fixed pipeline into an implicit general graph; and
|
|
||||||
- current code that no longer matches an accepted ADR or documented invariant.
|
|
||||||
|
|
||||||
Graph-reported cross-layer calls must be traced before being classified because
|
|
||||||
tests and interface implementations can resemble dependency inversions without
|
|
||||||
creating a production import violation.
|
|
||||||
|
|
||||||
### 2. Configuration And CLI Composition
|
|
||||||
|
|
||||||
Inspect `internal/core/config`, `internal/cli`, configuration parsing and
|
|
||||||
redaction tests, profile construction, session derivation, catalog assembly,
|
|
||||||
reference overrides, run-result handling, terminal reporting, and maintained
|
|
||||||
example contract tests.
|
|
||||||
|
|
||||||
Pay particular attention to the currently dense paths around
|
|
||||||
`runPipelineCommand`, configuration profile validation, selected reference
|
|
||||||
targets, recomputation policy, and option normalization. Determine whether
|
|
||||||
their complexity reflects necessary composition or mixed responsibilities that
|
|
||||||
can be separated without moving policy into the framework.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- file, environment, CLI, pipeline, binding, and prompt-default precedence;
|
|
||||||
- consistent strict option and unknown-field handling;
|
|
||||||
- session identity independence from references and pipeline-local changes;
|
|
||||||
- effective profile and runtime fingerprint consistency;
|
|
||||||
- redaction before errors or debug/manifest boundaries;
|
|
||||||
- output publication only after framework success; and
|
|
||||||
- one guarded terminalization path that preserves the primary failure.
|
|
||||||
|
|
||||||
### 3. Pipeline Resolution, Preparation, And Typed Registries
|
|
||||||
|
|
||||||
Inspect `internal/framework/pipeline/profile.go`, `prepare.go`, registry files,
|
|
||||||
`options.go`, `references.go`, `handoff.go`, `construction.go`, typed contracts,
|
|
||||||
and their focused tests.
|
|
||||||
|
|
||||||
This area deserves a dedicated pass because the current graph identifies
|
|
||||||
`ResolvePipeline`, generated-binding validation, reference-target resolution,
|
|
||||||
and generated-reference construction as high-complexity or high-fan-in code.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- static failures occur before source parsing;
|
|
||||||
- selected and unselected lanes do not contaminate each other's requirements;
|
|
||||||
- stage defaults and overrides have one canonical resolution path;
|
|
||||||
- typed registration and private erasure cannot panic or accept near-matching
|
|
||||||
artifact types;
|
|
||||||
- generated bindings reject cycles, forward references, ambiguity, wrong kinds,
|
|
||||||
and missing accepted normalized producers;
|
|
||||||
- materialized reference bytes and options are cloned and bounded; and
|
|
||||||
- resolved composition and prepared fingerprints include the complete semantic
|
|
||||||
policy exactly once.
|
|
||||||
|
|
||||||
Compare input, chunker, extractor, merger, normalizer, output, validator, codec,
|
|
||||||
evidence-projector, and validator-chain registries for shared mechanics and
|
|
||||||
intentional differences. Repeated typed registration code is a candidate only
|
|
||||||
if a helper can retain useful compile-time guarantees and stage-specific
|
|
||||||
diagnostics.
|
|
||||||
|
|
||||||
### 4. Pipeline Execution, Validation, Retry, And Concurrency
|
|
||||||
|
|
||||||
Inspect `runner*.go`, `typed_execution.go`, `runner_typed.go`,
|
|
||||||
`runner_concurrent.go`, validation-chain execution, normalize retry behavior,
|
|
||||||
synchronized collaborators, and the concurrency, cancellation, retry, debug,
|
|
||||||
and checkpoint tests.
|
|
||||||
|
|
||||||
Trace complete paths rather than reviewing helper files in isolation:
|
|
||||||
|
|
||||||
- source and chunk-plan selection through chunk validation;
|
|
||||||
- deterministic chunk-first/lane-second dispatch;
|
|
||||||
- lane extraction through merge and normalize continuations;
|
|
||||||
- rejection versus framework-error propagation;
|
|
||||||
- cancellation before dispatch, while queued, and while running;
|
|
||||||
- retry attempts and warning retention;
|
|
||||||
- stable error selection after concurrent completion;
|
|
||||||
- checkpoint hydration back into typed execution; and
|
|
||||||
- output suppression after a framework error.
|
|
||||||
|
|
||||||
Look for goroutine leaks, unbounded work, lock-order risks, double release or
|
|
||||||
double recording, races on shared result state, unnecessary serialization,
|
|
||||||
and repeated canonicalization. Comments are especially valuable here when they
|
|
||||||
explain ordering or cancellation invariants that are not apparent from local
|
|
||||||
control flow.
|
|
||||||
|
|
||||||
### 5. State, Checkpoints, Chunk Plans, Debugging, And File Safety
|
|
||||||
|
|
||||||
Inspect `internal/framework/checkpoint`, `chunkplan`, `chunkmap`, `debug`,
|
|
||||||
`evidencecontext`, `internal/core/fileio`, `debugbundle`, and their CLI
|
|
||||||
composition.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- narrow path validation and symlink-resistant confinement;
|
|
||||||
- atomic writes and recoverable explicit cleanup;
|
|
||||||
- separation of checkpoint recording, resume loading, chunk-plan caching, and
|
|
||||||
debug capture;
|
|
||||||
- canonical encoding before content identity is trusted;
|
|
||||||
- complete but non-secret checkpoint fingerprints;
|
|
||||||
- correct ordinary-resume and selective-recompute behavior;
|
|
||||||
- producer dependency invalidation across ordered steps;
|
|
||||||
- immutable hydration and no aliasing with stored bytes;
|
|
||||||
- debug data never influencing execution or reuse; and
|
|
||||||
- terminal persistence failures never obscuring the primary error.
|
|
||||||
|
|
||||||
Review the repeated extract/merge/normalize recorder and loader methods, path
|
|
||||||
component validators in multiple state packages, and clone/encode/decode paths.
|
|
||||||
Determine which repetition is a clear stage adapter and which can share a
|
|
||||||
private primitive without weakening reason-code ownership or diagnostics.
|
|
||||||
|
|
||||||
### 6. LLM Runtime, Prompt Filesystems, And Assets
|
|
||||||
|
|
||||||
Inspect `internal/framework/llm`, `promptfs`, the PromptKit integration,
|
|
||||||
scheduler, profile-source construction, prompt/schema registries, root
|
|
||||||
`assets`, module prompt manifests, and relevant D&D shared assets.
|
|
||||||
|
|
||||||
Verify:
|
|
||||||
|
|
||||||
- every provider call passes through the shared scheduler and cancellation
|
|
||||||
removes queued calls safely;
|
|
||||||
- PromptKit and Notarius concurrency limits compose as documented;
|
|
||||||
- profile inspection and runtime use identical source precedence;
|
|
||||||
- session IDs, profile-source fingerprints, prompt fingerprints, and schema
|
|
||||||
fingerprints reflect the intended semantic inputs;
|
|
||||||
- secrets and provider-specific error types do not cross the boundary;
|
|
||||||
- prompt inputs and private outputs do not expose opaque entity IDs;
|
|
||||||
- prompt ordering, stable prefixes, and cache controls remain intentional;
|
|
||||||
- schema loaders and filesystem adapters validate once and return defensive
|
|
||||||
data; and
|
|
||||||
- the root assets package contains no business logic.
|
|
||||||
|
|
||||||
Compare the LLM asset registry and prompt-filesystem adapters for duplicated
|
|
||||||
filesystem behavior. Review repeated prompt/schema loader and metadata code in
|
|
||||||
module packages, but reject an extraction that would centralize domain prompt
|
|
||||||
ownership or make unrelated assets share one invalidation boundary.
|
|
||||||
|
|
||||||
### 7. Generic And Seriatim Modules
|
|
||||||
|
|
||||||
Inspect `internal/modules/generic` and `internal/modules/seriatim`, including
|
|
||||||
module specs, option decoding, chunk planning, input translation, validators,
|
|
||||||
output encoding, evidence-context publication, registration, and tests.
|
|
||||||
|
|
||||||
Verify that:
|
|
||||||
|
|
||||||
- external Seriatim details end at the input boundary;
|
|
||||||
- generic chunking and output remain domain-neutral;
|
|
||||||
- chunk plans and source units preserve source-addressed invariants;
|
|
||||||
- output logical names are safe and deterministic;
|
|
||||||
- output options do not bypass preparation-time compatibility checks; and
|
|
||||||
- option decoding is strict, small, and consistent with configuration
|
|
||||||
validation.
|
|
||||||
|
|
||||||
The graph flags generic integer option parsing and JSON output policy decoding
|
|
||||||
as relatively complex. Examine whether that is inherent strict decoding or an
|
|
||||||
opportunity for a smaller typed parser with equally precise diagnostics.
|
|
||||||
|
|
||||||
### 8. D&D Domain Model, Codecs, And Shared Helpers
|
|
||||||
|
|
||||||
Inspect `internal/modules/dnd` domain types, codecs, candidate decoders,
|
|
||||||
identity packages, registries, shared source-reference helpers, diagnostics,
|
|
||||||
registry resolution, entity reconciliation, mergers, registrar, and assets.
|
|
||||||
|
|
||||||
Compare all ten current artifact families. Build a convention matrix covering:
|
|
||||||
|
|
||||||
- module specs and execution classes;
|
|
||||||
- constructor and option behavior;
|
|
||||||
- manifest metadata and checkpoint fingerprints;
|
|
||||||
- response-schema loading and private-versus-durable types;
|
|
||||||
- source-reference conversion, canonicalization, ordering, and deduplication;
|
|
||||||
- nil versus present-empty output;
|
|
||||||
- codecs and strict JSON behavior;
|
|
||||||
- registry lookup, identity derivation, immutable projections, and resolution;
|
|
||||||
- normalizer retry/fallback behavior;
|
|
||||||
- validators and default chains; and
|
|
||||||
- registration, prompt assets, and documentation ownership.
|
|
||||||
|
|
||||||
The graph reports many exact similarities among codec `Decode` methods,
|
|
||||||
fingerprint/metadata methods, registry extractors, identity helpers, occurrence
|
|
||||||
normalizers, and validators. Treat these as a prioritized review list, not an
|
|
||||||
instruction to create one generic D&D engine. A worthwhile helper must preserve
|
|
||||||
domain-specific identity, evidence, kind ordering, validation, diagnostics,
|
|
||||||
and artifact typing.
|
|
||||||
|
|
||||||
### 9. D&D Extraction And Normalization Flows
|
|
||||||
|
|
||||||
Trace each lane end to end rather than auditing only similarly named files:
|
|
||||||
|
|
||||||
- spells;
|
|
||||||
- NPC registry and NPC occurrences;
|
|
||||||
- combat turns and enemy events;
|
|
||||||
- item registry and item occurrences;
|
|
||||||
- scene descriptions; and
|
|
||||||
- location registry and location occurrences.
|
|
||||||
|
|
||||||
For registry/occurrence pairs, verify the complete semantic boundary: the model
|
|
||||||
uses contextual evidence, deterministic code attaches opaque identity, registry
|
|
||||||
evidence does not become occurrence evidence, and unresolved or ambiguous
|
|
||||||
selections fail according to lane policy.
|
|
||||||
|
|
||||||
Review whether any lane resolves or canonicalizes the same entity, source
|
|
||||||
reference, or response twice; constructs unnecessary intermediate response
|
|
||||||
forms; performs repeated sorts or scans; or retains transitional paths. Compare
|
|
||||||
registry normalizers and occurrence normalizers for genuinely identical
|
|
||||||
mechanics, while keeping currency, same-name location, NPC ambiguity, spell
|
|
||||||
catalog, scene eligibility, and combat-specific policy with their owners.
|
|
||||||
|
|
||||||
### 10. Test Suite And Comment Coverage
|
|
||||||
|
|
||||||
Review the tests associated with every preceding area after understanding the
|
|
||||||
production contracts. This should be a cross-cutting pass, not a request to add
|
|
||||||
tests for every flagged function.
|
|
||||||
|
|
||||||
Identify:
|
|
||||||
|
|
||||||
- consequential unprotected invariants;
|
|
||||||
- duplicated policy assertions across layers;
|
|
||||||
- brittle tests coupled to internal constants, prompt prose, or private helper
|
|
||||||
shape;
|
|
||||||
- oversized test harnesses and repeated fixtures that obscure intent;
|
|
||||||
- race-sensitive code not exercised under `-race`;
|
|
||||||
- parsers, canonicalizers, and path handlers where fuzzing would address a real
|
|
||||||
input-space risk; and
|
|
||||||
- complex production code whose tests reveal an unclear ownership boundary.
|
|
||||||
|
|
||||||
Also identify necessarily complex symbols that lack a concise invariant-level
|
|
||||||
comment. Comment recommendations should name the exact symbol and the fact the
|
|
||||||
comment should explain; “add more comments” is not an actionable finding.
|
|
||||||
|
|
||||||
## Efficiency Evaluation
|
|
||||||
|
|
||||||
The audit should consider both runtime and maintenance efficiency.
|
|
||||||
|
|
||||||
For runtime efficiency, examine algorithmic behavior relative to realistic
|
|
||||||
input dimensions: source units, chunks, lanes, references, artifacts, registry
|
|
||||||
records, checkpoint files, and prompt assets. Prioritize repeated full-input
|
|
||||||
passes, nested linear lookup, unnecessary JSON round trips, repeated hashing,
|
|
||||||
large defensive copies at adjacent ownership boundaries, and serialization on
|
|
||||||
concurrent hot paths. Preserve a defensive copy when it establishes ownership;
|
|
||||||
removing it solely to reduce allocation is not an improvement.
|
|
||||||
|
|
||||||
For maintenance efficiency, prioritize repeated policy, parallel type systems,
|
|
||||||
duplicated error classification, scattered defaults, and migrations that left
|
|
||||||
two ways to perform the same operation. Boilerplate is costly only when it can
|
|
||||||
drift or obscures the semantic core. Small explicit typed adapters can be more
|
|
||||||
maintainable than a generic abstraction.
|
|
||||||
|
|
||||||
Do not recommend caching, pooling, concurrency, or a benchmark without naming
|
|
||||||
the workload and risk it addresses. Add a benchmark only when a proposed
|
|
||||||
optimization concerns a repeatable local path and the result would influence
|
|
||||||
the decision.
|
|
||||||
|
|
||||||
## Audit Method
|
|
||||||
|
|
||||||
Each area should use the same method:
|
|
||||||
|
|
||||||
1. Read its architecture/internal documentation and focused tests.
|
|
||||||
2. Map public/package contracts and trace the main call paths.
|
|
||||||
3. Inspect high-fan-in, high-cognitive-complexity, nested-loop, and repeated-
|
|
||||||
conversion symbols.
|
|
||||||
4. Review exact and near-duplicate code side by side, including callers and
|
|
||||||
failure semantics.
|
|
||||||
5. Check dependency direction, ownership, aliasing, deterministic order,
|
|
||||||
cancellation, and error classification.
|
|
||||||
6. Compare tests with the risks owned at that layer.
|
|
||||||
7. Record findings and inspected-with-no-finding areas before moving on.
|
|
||||||
8. Run focused read-only validation when it can confirm or refute a suspected
|
|
||||||
problem.
|
|
||||||
|
|
||||||
Prefer the repository knowledge graph for symbol discovery, call tracing, and
|
|
||||||
similarity candidates. Use textual search for literals, diagnostics, config
|
|
||||||
keys, asset content, and stale names. Read complete implementations and tests
|
|
||||||
before reporting a metric-derived candidate.
|
|
||||||
|
|
||||||
## Execution Sequence
|
|
||||||
|
|
||||||
This document owns audit scope, questions, evidence standards, and the quality
|
|
||||||
bar. [Staged Codebase Audit Sequence](audit-sequence.md) is the sole canonical
|
|
||||||
owner of prompt order, stage boundaries, per-stage reading, validation commands,
|
|
||||||
and acceptance criteria. Do not derive or maintain a second sequence here.
|
|
||||||
|
|
||||||
The audit is executed as bounded prompts and writes its accumulated findings to
|
|
||||||
`docs/roadmap/audit.md`. Later stages must build on and reconcile earlier
|
|
||||||
evidence rather than concatenate independent reports. Implementation and
|
|
||||||
roadmap retirement remain separate work after maintainers review the completed
|
|
||||||
audit.
|
|
||||||
|
|
||||||
## Baseline And Validation
|
|
||||||
|
|
||||||
Before the first audit stage, record the commit under review and require a clean
|
|
||||||
worktree. Refresh the code knowledge graph so renamed or deleted code does not
|
|
||||||
produce false findings. Run the normal offline baseline:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./cmd/notarius
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Run `go test -race` for packages with concurrency or mutable shared state,
|
|
||||||
especially `internal/framework/pipeline`, `internal/framework/llm`, state
|
|
||||||
packages, and D&D registries. A repository-wide race run is appropriate for
|
|
||||||
final verification if its cost remains reasonable.
|
|
||||||
|
|
||||||
Optional diagnostic commands should be used only when relevant:
|
|
||||||
|
|
||||||
- `go test -count=1` to rule out cache-masked failures;
|
|
||||||
- `go test -shuffle=on` to detect order coupling;
|
|
||||||
- focused fuzzing for existing or newly justified fuzz targets; and
|
|
||||||
- focused benchmarks or profiles for a specific efficiency finding.
|
|
||||||
|
|
||||||
The audit itself should not change tests to make the baseline pass. Record any
|
|
||||||
pre-existing failure and distinguish it from an audit finding.
|
|
||||||
|
|
||||||
## Deliverable Quality Bar
|
|
||||||
|
|
||||||
The completed audit should:
|
|
||||||
|
|
||||||
- cover every repository area listed above;
|
|
||||||
- distinguish defects from refactoring opportunities and comment requests;
|
|
||||||
- distinguish credible performance costs from aesthetic simplification;
|
|
||||||
- identify intentional duplication that should remain explicit;
|
|
||||||
- avoid recommendations that violate dependency direction or weaken typing;
|
|
||||||
- cite exact evidence and preserve named invariants for every finding;
|
|
||||||
- consolidate root causes rather than report many symptoms;
|
|
||||||
- rank independent work so a later implementation plan can stage it safely;
|
|
||||||
- recommend no code change whose expected benefit is smaller than its added
|
|
||||||
abstraction or test-maintenance cost; and
|
|
||||||
- leave implementation and roadmap retirement to later work.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None are required to begin the audit. If a later stage cannot determine whether
|
|
||||||
behavior is intentional from code, tests, policies, ADRs, or current
|
|
||||||
documentation, it should record the uncertainty and a recommended resolution
|
|
||||||
rather than silently treating preference as a defect.
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,350 +0,0 @@
|
|||||||
# Contextual Entity Grounding
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Notarius should use an LLM for semantic interpretation of source evidence, not
|
|
||||||
for referential-integrity work that deterministic code can perform more
|
|
||||||
reliably. D&D prompts must therefore stop requiring models to reproduce opaque
|
|
||||||
machine identifiers such as hash-derived entity IDs. Models should identify
|
|
||||||
entities through human-readable, evidence-grounded context, after which
|
|
||||||
Notarius resolves the selection and attaches the canonical durable identity.
|
|
||||||
|
|
||||||
This roadmap defines the policy, affected D&D prompt families, and intended
|
|
||||||
end state. The ordered work needed to reach that state is maintained in
|
|
||||||
[Implementation Plan](implementation.md).
|
|
||||||
|
|
||||||
## User Intent
|
|
||||||
|
|
||||||
The change has two goals:
|
|
||||||
|
|
||||||
- prevent otherwise useful model responses from failing because a long,
|
|
||||||
non-semantic string was copied incorrectly; and
|
|
||||||
- avoid spending prompt space and model effort on exact-copy work that provides
|
|
||||||
no semantic value.
|
|
||||||
|
|
||||||
The policy is not a ban on identifiers. Durable artifacts may continue to use
|
|
||||||
application-owned IDs, and prompts may continue to request source-unit ranges
|
|
||||||
that locate evidence. The policy governs which identity work is assigned to
|
|
||||||
the model.
|
|
||||||
|
|
||||||
## Policy
|
|
||||||
|
|
||||||
An LLM-facing prompt input or private response schema must not require a model
|
|
||||||
to reproduce an opaque machine identifier when Notarius can establish the same
|
|
||||||
association deterministically.
|
|
||||||
|
|
||||||
Opaque machine identifiers include cryptographic hashes, UUIDs, digests,
|
|
||||||
database keys, durable entity IDs, and other tokens whose characters do not
|
|
||||||
carry source-grounded meaning for the model. These values may remain in
|
|
||||||
application state, provenance, diagnostics, checkpoints, and durable artifact
|
|
||||||
contracts, but should be omitted from model-visible material when they do not
|
|
||||||
help the model make a semantic decision.
|
|
||||||
|
|
||||||
The intended responsibility boundary is:
|
|
||||||
|
|
||||||
- the model decides which contextual entity is supported by the supplied
|
|
||||||
evidence and returns the bounded semantic facts requested by the module;
|
|
||||||
- the calling module validates that the contextual selection resolves to
|
|
||||||
exactly one supplied candidate;
|
|
||||||
- deterministic code supplies the canonical display value and durable entity
|
|
||||||
ID; and
|
|
||||||
- existing validators continue to enforce referential integrity at later
|
|
||||||
artifact boundaries.
|
|
||||||
|
|
||||||
Transcript `start_unit_id` and `end_unit_id` values are permitted. They are
|
|
||||||
contextual source coordinates and form part of the evidence contract rather
|
|
||||||
than arbitrary identity tokens. Prompt IDs, schema IDs, fingerprints, session
|
|
||||||
IDs, and digests may also remain in runtime metadata that the model is not
|
|
||||||
asked to reproduce.
|
|
||||||
|
|
||||||
Short request-local labels are a narrowly permitted fallback only when a
|
|
||||||
contextual selector cannot uniquely represent the available choices without
|
|
||||||
unreasonable prompt cost. Such a label must be compact, scoped to one request,
|
|
||||||
validated against the supplied candidate set, and never reused as a durable
|
|
||||||
identity. Current D&D occurrence and reconciliation prompts should be designed
|
|
||||||
without this exception; adopting it later requires a concrete demonstrated
|
|
||||||
need and documented rationale.
|
|
||||||
|
|
||||||
## Current State
|
|
||||||
|
|
||||||
The initial NPC, item, and location registry extractors already follow the
|
|
||||||
desired pattern: the model returns contextual names and evidence, and Notarius
|
|
||||||
derives durable IDs afterward. Spells, combat turns, and enemy events use
|
|
||||||
contextual actor names rather than requiring hash-derived NPC IDs.
|
|
||||||
|
|
||||||
Two current prompt families diverge from that pattern:
|
|
||||||
|
|
||||||
1. `dnd/npc-occurrences`, `dnd/item-occurrences`, and
|
|
||||||
`dnd/location-occurrences` place durable registry IDs in model-visible
|
|
||||||
projections and require the private LLM response to repeat those IDs.
|
|
||||||
2. NPC-, item-, and location-registry normalization use the shared entity
|
|
||||||
reconciliation prompt, which labels candidates with opaque
|
|
||||||
`candidate-000001`-style keys and requires the model to copy those keys into
|
|
||||||
duplicate-group proposals.
|
|
||||||
|
|
||||||
The durable occurrence artifacts correctly retain canonical entity IDs. The
|
|
||||||
problem is the private model transport contract, not the published artifact
|
|
||||||
contract.
|
|
||||||
|
|
||||||
## Target Architecture
|
|
||||||
|
|
||||||
### Model proposals and durable artifacts
|
|
||||||
|
|
||||||
Private LLM response types must express contextual semantic proposals rather
|
|
||||||
than reuse the durable artifact type when that type contains an opaque entity
|
|
||||||
ID. The extractor maps a validated private response into the existing durable
|
|
||||||
artifact only after identity resolution succeeds.
|
|
||||||
|
|
||||||
No affected durable artifact kind, media type, schema ID, schema version, or
|
|
||||||
JSON field changes as part of this work. NPC, item, and location occurrence
|
|
||||||
artifacts continue to publish their exact canonical ID/name pair. Registry
|
|
||||||
artifacts likewise retain their IDs and evidence.
|
|
||||||
|
|
||||||
The private schemas and prompt declarations may remain at their current `v1`
|
|
||||||
identities because Notarius is pre-release and these are not external
|
|
||||||
contracts. Their content hashes, mapping-policy fingerprints, and affected
|
|
||||||
prompt fingerprints must change so incompatible checkpoints are not reused.
|
|
||||||
|
|
||||||
### NPC occurrence grounding
|
|
||||||
|
|
||||||
The NPC occurrence prompt receives an ordered names-only projection of the
|
|
||||||
normalized NPC registry. Its private response contains the canonical NPC name,
|
|
||||||
occurrence kind, and current-transcript source ranges, but no `npc_id`.
|
|
||||||
|
|
||||||
The extractor resolves the returned name under the existing NPC comparison
|
|
||||||
policy. Resolution must produce exactly one registry entry. It then writes that
|
|
||||||
entry's canonical display name and durable ID into the `dnd.NPCOccurrence`.
|
|
||||||
An unknown or ambiguous selection invalidates the extraction operation; the
|
|
||||||
extractor must not guess, use fuzzy matching, silently omit the record, or
|
|
||||||
accept a partial response.
|
|
||||||
|
|
||||||
### Item occurrence grounding
|
|
||||||
|
|
||||||
The item occurrence prompt receives an ordered names-only projection of the
|
|
||||||
normalized item registry. Its private response contains the canonical item
|
|
||||||
name, occurrence kind, kind-specific fields, and current-transcript source
|
|
||||||
ranges, but no `item_id`.
|
|
||||||
|
|
||||||
The extractor resolves the returned name under the existing item comparison
|
|
||||||
and identity policies. Resolution must produce exactly one registry entry,
|
|
||||||
whose canonical name and durable ID are attached deterministically. Unknown or
|
|
||||||
ambiguous selections invalidate the complete extraction operation rather than
|
|
||||||
being guessed, repaired by similarity, or dropped.
|
|
||||||
|
|
||||||
### Location occurrence grounding
|
|
||||||
|
|
||||||
Location identity cannot always be resolved from a display name alone: the
|
|
||||||
current registry intentionally permits same-name locations with distinct
|
|
||||||
source anchors. The location occurrence prompt must therefore receive a
|
|
||||||
contextual registry descriptor that contains the canonical display name plus
|
|
||||||
the minimum source-grounded registry evidence needed to distinguish same-name
|
|
||||||
records. It must not contain the durable `location:sha256:...` value.
|
|
||||||
|
|
||||||
The private response uses two required selector fields: `name` and
|
|
||||||
`registry_refs`. For a comparison-unique canonical name, `registry_refs` is an
|
|
||||||
empty array and Notarius resolves the name under the location comparison
|
|
||||||
policy. For a name shared by multiple registry records, `registry_refs`
|
|
||||||
contains that record's complete canonically ordered registry ranges as
|
|
||||||
`start_unit_id` and `end_unit_id` pairs, without `source_id`.
|
|
||||||
|
|
||||||
Every model-facing registry entry uses one fixed shape with required `name`,
|
|
||||||
`registry_refs`, and `context` fields. `context` is an array of strict objects
|
|
||||||
containing only `unit_id` and `text`. Comparison-unique entries use empty
|
|
||||||
`registry_refs` and `context` arrays. Same-name entries use the complete
|
|
||||||
registry-range selector and the bounded context described below. The model
|
|
||||||
returns only `name` and `registry_refs`; it does not reproduce `context`.
|
|
||||||
|
|
||||||
For same-name groups, the projection also supplies bounded transcript units
|
|
||||||
covered by each record's registry ranges so the model receives meaningful
|
|
||||||
identity context rather than coordinates alone. Those ranges must resolve
|
|
||||||
against the current source document, and the resulting contextual selectors
|
|
||||||
must be unique. An invalid range or selector collision prevents the LLM call
|
|
||||||
and fails the operation. Unique-name entries do not repeat registry ranges or
|
|
||||||
context in the selector, preserving compatibility with a valid registry from
|
|
||||||
another source when the name alone is unambiguous.
|
|
||||||
|
|
||||||
The private response separately supplies current-transcript `source_refs` that
|
|
||||||
prove the occurrence. Registry identity evidence and occurrence evidence must
|
|
||||||
remain different fields and must never be merged. The model should omit an
|
|
||||||
occurrence when the transcript does not support choosing among same-name
|
|
||||||
locations. If a returned selector does not resolve to exactly one supplied
|
|
||||||
registry record, the extractor invalidates the complete operation rather than
|
|
||||||
guessing.
|
|
||||||
|
|
||||||
### Registry reconciliation
|
|
||||||
|
|
||||||
The shared entity-reconciliation input replaces opaque candidate keys with
|
|
||||||
contextual candidate descriptors. At minimum, a descriptor contains the
|
|
||||||
candidate's display name and its canonically ordered source-reference ranges;
|
|
||||||
the existing transcript windows remain available for semantic judgment.
|
|
||||||
|
|
||||||
Duplicate-group members and the canonical member in the private response use
|
|
||||||
the same contextual descriptor shape. The shared reconciliation helper maps
|
|
||||||
each descriptor back to exactly one internal candidate before assessing the
|
|
||||||
proposal. Exact deterministic duplicates should already be removed before the
|
|
||||||
LLM call; any remaining descriptor collision makes the affected candidate
|
|
||||||
ineligible for model-assisted reconciliation rather than authorizing an
|
|
||||||
arbitrary choice.
|
|
||||||
|
|
||||||
Existing safety behavior remains in force: groups must contain at least two
|
|
||||||
supplied candidates, the canonical candidate must be a member, groups must not
|
|
||||||
overlap, and domain-specific eligibility rules remain authoritative. Invalid,
|
|
||||||
ambiguous, or unsafe groups are discarded through the existing bounded
|
|
||||||
fallback and diagnostic behavior. The model never directly mutates the
|
|
||||||
durable registry.
|
|
||||||
|
|
||||||
The shared private reconciliation schema and helper must remain domain-neutral
|
|
||||||
within the D&D family. NPC-, item-, and location-specific duplicate policy
|
|
||||||
continues to live in the owning normalizer.
|
|
||||||
|
|
||||||
## Prompt And Asset Changes
|
|
||||||
|
|
||||||
The following LLM-facing assets are in scope:
|
|
||||||
|
|
||||||
- the prompt instructions, registry input fragments, and private response
|
|
||||||
schemas for NPC, item, and location occurrences;
|
|
||||||
- the prompt manifests where input shape or selected fragments change;
|
|
||||||
- the shared D&D entity-reconciliation fragment and private response schema;
|
|
||||||
and
|
|
||||||
- the NPC-, item-, and location-registry normalization prompt inputs that use
|
|
||||||
the shared reconciliation contract.
|
|
||||||
|
|
||||||
Affected projections must exclude durable entity IDs rather than merely stop
|
|
||||||
mentioning them in prose. Prompt instructions should describe the contextual
|
|
||||||
selection rule once at the narrowest owning asset and must preserve the current
|
|
||||||
distinction between registry grounding and transcript evidence.
|
|
||||||
|
|
||||||
Prompt ordering and cache controls should remain unchanged unless the new
|
|
||||||
contextual input requires an intentional manifest change. Unrelated shared
|
|
||||||
prompt bytes should not be edited. Prompt and schema fingerprints should
|
|
||||||
invalidate only the operations whose selected assets or mapping semantics
|
|
||||||
changed.
|
|
||||||
|
|
||||||
## Code And Validation Changes
|
|
||||||
|
|
||||||
The occurrence extractors need private response types and deterministic
|
|
||||||
registry-resolution paths appropriate to their domain. Shared code is
|
|
||||||
appropriate only for demonstrated mechanics that have identical semantics;
|
|
||||||
NPC, item, and location ambiguity policies must not be forced behind a generic
|
|
||||||
resolver merely to reduce line count.
|
|
||||||
|
|
||||||
Registry projections should expose explicit model-facing methods whose names
|
|
||||||
describe whether they are names-only or contextual identity projections. The
|
|
||||||
existing ID/name projections may remain only for deterministic consumers that
|
|
||||||
genuinely require them; they must no longer be wired to an LLM input.
|
|
||||||
|
|
||||||
Mapping-policy and normalization-policy identifiers must be reviewed and
|
|
||||||
advanced wherever their semantics change. Checkpoint fingerprints must cover
|
|
||||||
the new projection content, private schema, prompt assets, and mapping policy,
|
|
||||||
while continuing to exclude irrelevant internal implementation details.
|
|
||||||
|
|
||||||
Durable occurrence normalizers and registry validators remain defense in
|
|
||||||
depth. They continue to validate exact ID/name pairs on artifacts entering
|
|
||||||
through checkpoints, codecs, or other boundaries even though the LLM no longer
|
|
||||||
produces the ID directly.
|
|
||||||
|
|
||||||
## Testing And Evaluation
|
|
||||||
|
|
||||||
Tests should protect the behavioral boundary rather than prompt prose or
|
|
||||||
private helper structure. The completed work should demonstrate that:
|
|
||||||
|
|
||||||
- affected model-facing registry projections do not contain durable entity
|
|
||||||
IDs;
|
|
||||||
- private occurrence schemas reject opaque ID fields and accept the intended
|
|
||||||
contextual shape;
|
|
||||||
- valid contextual selections map to the exact canonical durable ID/name pair;
|
|
||||||
- unknown, mismatched, and ambiguous selections fail without fuzzy matching,
|
|
||||||
partial acceptance, or arbitrary reassignment;
|
|
||||||
- same-name locations remain distinguishable through contextual evidence;
|
|
||||||
- reconciliation preserves equal-name candidates, resolves valid contextual
|
|
||||||
groups, and discards ambiguous or unsafe proposals;
|
|
||||||
- registry evidence never becomes occurrence evidence;
|
|
||||||
- durable codec, normalization, and validator behavior remains compatible; and
|
|
||||||
- representative assembled D&D pipelines still prepare and execute with fake
|
|
||||||
structured-LLM responses.
|
|
||||||
|
|
||||||
Do not add repository-wide prompt-prose snapshots, exact-message-count tests,
|
|
||||||
or a change-detector test that merely scans for today's field names. Focused
|
|
||||||
projection, schema, mapping, fallback, and integration tests are the stable
|
|
||||||
owners of these risks. Model-quality evaluation with representative
|
|
||||||
transcripts remains a manual development aid rather than an offline test gate.
|
|
||||||
|
|
||||||
## Documentation And Architectural Record
|
|
||||||
|
|
||||||
This policy is durable and applies to future modules, so it warrants
|
|
||||||
`docs/adr/0012-resolve-opaque-entity-identifiers-deterministically.md`, which
|
|
||||||
records:
|
|
||||||
|
|
||||||
- the semantic-proposal versus referential-integrity boundary;
|
|
||||||
- why durable opaque IDs are excluded from model response contracts;
|
|
||||||
- why contextual evidence coordinates remain permitted;
|
|
||||||
- the narrowly scoped request-local-label exception;
|
|
||||||
- alternatives including durable IDs, names-only matching, and short opaque
|
|
||||||
handles; and
|
|
||||||
- the consequences for private schemas, deterministic resolution, debugging,
|
|
||||||
and ambiguous identities.
|
|
||||||
|
|
||||||
`docs/policy/architecture.md` states the general LLM boundary invariant and
|
|
||||||
links to the ADR. `docs/internal/dnd.md` describes the concrete occurrence
|
|
||||||
projections, contextual reconciliation selectors, resolution and failure
|
|
||||||
behavior, and the continued separation of registry grounding from occurrence
|
|
||||||
evidence. `docs/internal/llm.md` contains only a short clarification that
|
|
||||||
caller-owned modules, not PromptKit or the transport adapter, resolve
|
|
||||||
contextual model selections into application identities.
|
|
||||||
|
|
||||||
The NPC, item, and location occurrence and registry integration documents must
|
|
||||||
continue to own their durable wire contracts, while removing current claims
|
|
||||||
that the model-facing consumer projection contains `{id,name}` or that the raw
|
|
||||||
LLM response supplies the durable ID. They should instead explain that
|
|
||||||
Notarius resolves contextual model output and publishes the same exact durable
|
|
||||||
ID/name pair. No public schema examples need to remove those IDs.
|
|
||||||
|
|
||||||
The generic LLM-assisted deduplication entry in `docs/roadmap/future.md` must be
|
|
||||||
reconciled with this policy: stable IDs may exist inside deterministic state,
|
|
||||||
but a future model-facing proposal should use contextual selectors or a
|
|
||||||
documented request-local-label exception rather than durable IDs.
|
|
||||||
|
|
||||||
## Compatibility And Operational Effects
|
|
||||||
|
|
||||||
This work intentionally changes private prompt inputs, private structured
|
|
||||||
responses, and mapping semantics. It will invalidate affected checkpoints
|
|
||||||
through existing prompt, schema, projection, and policy fingerprints. No
|
|
||||||
manual checkpoint migration is required.
|
|
||||||
|
|
||||||
Durable D&D artifacts and generated-reference compatibility remain unchanged.
|
|
||||||
Operators do not receive new configuration fields or CLI controls. The feature
|
|
||||||
does not change PromptKit, provider routing, profile selection, retries,
|
|
||||||
concurrency, or public output placement.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This work does not:
|
|
||||||
|
|
||||||
- remove canonical IDs from durable registries or occurrence artifacts;
|
|
||||||
- change occurrence categories, evidence rules, or registry identity policy;
|
|
||||||
- add fuzzy, probabilistic, or embedding-based entity resolution;
|
|
||||||
- allow registry provenance to substitute for occurrence evidence;
|
|
||||||
- introduce a general entity graph or cross-artifact identity framework;
|
|
||||||
- redesign unrelated D&D prompts or their schemas;
|
|
||||||
- implement the future generic deduplication normalizer; or
|
|
||||||
- add provider-specific prompt behavior.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
The target state is complete when:
|
|
||||||
|
|
||||||
- no maintained D&D prompt requires a model to reproduce a durable opaque
|
|
||||||
entity ID;
|
|
||||||
- current D&D reconciliation prompts no longer require opaque candidate keys;
|
|
||||||
- NPC, item, and location occurrence LLM outputs are resolved
|
|
||||||
deterministically into their unchanged durable artifacts;
|
|
||||||
- same-name location and reconciliation cases remain safe and unambiguous;
|
|
||||||
- invalid contextual selections preserve the existing extraction-failure or
|
|
||||||
normalization-fallback semantics appropriate to their stage;
|
|
||||||
- affected checkpoint identities change without altering public schema
|
|
||||||
versions;
|
|
||||||
- focused and repository-wide tests pass offline;
|
|
||||||
- the ADR, architecture invariant, D&D internal guide, LLM internal guide,
|
|
||||||
relevant integration contracts, and future roadmap accurately describe
|
|
||||||
their canonical portions of the implemented policy; and
|
|
||||||
- no unrelated code, prompt behavior, or public contract changes are included.
|
|
||||||
224
docs/roadmap/dnd-subprocess-documentation.md
Normal file
224
docs/roadmap/dnd-subprocess-documentation.md
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
# D&D Subprocess Consumer Documentation
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Completed. The target guide is `docs/consumers/dnd-pipeline.md`.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Provide one task-oriented guide for applications that run Notarius as a
|
||||||
|
subprocess to execute the maintained complete D&D pipeline and consume its
|
||||||
|
published artifacts. The initial concrete consumer is Narratio, but the guide
|
||||||
|
must describe the public Notarius workflow rather than depend on Narratio
|
||||||
|
internals.
|
||||||
|
|
||||||
|
The guide should make the safe integration path obvious without duplicating
|
||||||
|
the CLI, input, receipt, output-bundle, or individual artifact contracts that
|
||||||
|
already have canonical documentation.
|
||||||
|
|
||||||
|
## Current State
|
||||||
|
|
||||||
|
The public integration surface is documented accurately but is distributed
|
||||||
|
across several documents:
|
||||||
|
|
||||||
|
- `docs/consumers/subprocess.md` defines the generic subprocess workflow;
|
||||||
|
- `docs/cli.md` owns commands, flags, stream behavior, and exit statuses;
|
||||||
|
- `docs/integrations/seriatim.md` owns the accepted transcript input shape;
|
||||||
|
- `docs/integrations/run-result.md` owns the machine-readable successful-run
|
||||||
|
receipt;
|
||||||
|
- `docs/integrations/json-output.md` owns bundle discovery and logical files;
|
||||||
|
- the D&D integration documents own the individual lane payload contracts;
|
||||||
|
- `examples/dnd-complete.config.yml` is the maintained complete pipeline.
|
||||||
|
|
||||||
|
A consumer can reconstruct the full workflow from those documents, but there
|
||||||
|
is no D&D-focused guide that connects the maintained example to its input,
|
||||||
|
invocation, complete artifact inventory, discovery procedure, and downstream
|
||||||
|
acceptance decisions.
|
||||||
|
|
||||||
|
## Target Documentation Set
|
||||||
|
|
||||||
|
### Create `docs/consumers/dnd-pipeline.md`
|
||||||
|
|
||||||
|
This document should own the end-to-end consumer workflow for the maintained
|
||||||
|
complete D&D configuration. It should be useful to Narratio and to another
|
||||||
|
subprocess orchestrator with the same needs.
|
||||||
|
|
||||||
|
The guide should contain the following sections.
|
||||||
|
|
||||||
|
#### Prerequisites And Deployment Configuration
|
||||||
|
|
||||||
|
- Link to `examples/dnd-complete.config.yml` rather than embedding a second
|
||||||
|
complete configuration.
|
||||||
|
- Explain that a deployment must provide the configured PromptKit profile and
|
||||||
|
campaign reference files.
|
||||||
|
- Recommend absolute paths for a service or orchestrator deployment.
|
||||||
|
- Call out the path-resolution distinction explicitly: YAML reference paths
|
||||||
|
are relative to the Notarius configuration file, while
|
||||||
|
`promptkit.profile_file` is relative to the Notarius process working
|
||||||
|
directory.
|
||||||
|
- Recommend validating the selected configuration and `dnd-session` pipeline
|
||||||
|
before processing sessions.
|
||||||
|
|
||||||
|
#### Transcript Input
|
||||||
|
|
||||||
|
- State that the complete pipeline consumes a Seriatim JSON document.
|
||||||
|
- Link to the canonical Seriatim contract for required fields and validation.
|
||||||
|
- Recommend the caller's final trimmed transcript when the caller maintains
|
||||||
|
transcript tiers. For Narratio, identify the implemented source as
|
||||||
|
`narratio.transcript.final_trimmed`, normally stored at
|
||||||
|
`transcripts/final.trimmed.json`.
|
||||||
|
- Explain that segment IDs must remain stable because D&D source references
|
||||||
|
cite those units.
|
||||||
|
- Explain that Notarius derives its default prompt session from the input
|
||||||
|
module and exact input bytes and that ordinary callers should not supply
|
||||||
|
`--session-id`.
|
||||||
|
|
||||||
|
#### Subprocess Invocation
|
||||||
|
|
||||||
|
- Show one concise invocation using `notarius run dnd-session`, explicit
|
||||||
|
absolute `--config`, `--input`, and `--output-dir` paths, and `--json`.
|
||||||
|
- Direct callers to capture stdout and stderr separately, propagate
|
||||||
|
cancellation, impose an operator-appropriate timeout, and wait for process
|
||||||
|
completion before parsing stdout.
|
||||||
|
- State that only exit status zero permits receipt decoding and link to the CLI
|
||||||
|
contract for the complete exit-status definition.
|
||||||
|
- Recommend retaining stderr and the invocation context for diagnosis without
|
||||||
|
logging secrets or transcript content.
|
||||||
|
|
||||||
|
#### Receipt And Bundle Discovery
|
||||||
|
|
||||||
|
- Require callers to accept only supported run-result schema versions while
|
||||||
|
tolerating unknown fields allowed by that version.
|
||||||
|
- Direct callers to obtain the exact run-specific bundle from the receipt's
|
||||||
|
absolute `output_directory`; they must not scan for the newest run directory
|
||||||
|
or construct a run ID.
|
||||||
|
- Require a confinement check when resolving `index_file` beneath the reported
|
||||||
|
bundle root.
|
||||||
|
- Direct callers to discover lane payloads by `lane_id` in `index.json`, then
|
||||||
|
verify descriptor media type and schema identity before decoding them.
|
||||||
|
- Explain that descriptor paths are untrusted relative paths and require the
|
||||||
|
same confinement discipline.
|
||||||
|
|
||||||
|
#### Complete D&D Artifact Inventory
|
||||||
|
|
||||||
|
Include a compact table for the ten lane IDs selected by the maintained
|
||||||
|
complete configuration:
|
||||||
|
|
||||||
|
- `item-registry`;
|
||||||
|
- `npc-registry`;
|
||||||
|
- `location-registry`;
|
||||||
|
- `scene-descriptions`;
|
||||||
|
- `item-occurrences`;
|
||||||
|
- `spells`;
|
||||||
|
- `combat-turns`;
|
||||||
|
- `npc-occurrences`;
|
||||||
|
- `location-occurrences`;
|
||||||
|
- `enemy-events`.
|
||||||
|
|
||||||
|
For each row, give a one-line purpose and link to the corresponding canonical
|
||||||
|
D&D artifact contract. Do not copy its fields or schema rules into the
|
||||||
|
consumer guide.
|
||||||
|
|
||||||
|
Document the four always-published bundle files—`index.json`, `manifest.json`,
|
||||||
|
`rejected.json`, and `warnings.json`—and the complete example's configured
|
||||||
|
`chunk-map.json` and `evidence-context.json` pipeline-wide artifacts. Link to
|
||||||
|
their canonical contracts and distinguish pipeline-wide artifacts from lane
|
||||||
|
outputs.
|
||||||
|
|
||||||
|
The inventory must say that a file is available only when its corresponding
|
||||||
|
artifact was accepted and published. It must not imply that process success
|
||||||
|
guarantees every configured lane.
|
||||||
|
|
||||||
|
#### Downstream Acceptance And Retention
|
||||||
|
|
||||||
|
- Explain that exit status zero can coexist with rejected outputs, warnings,
|
||||||
|
or absent lane descriptors.
|
||||||
|
- Require the consumer to define its required lane set explicitly. Recommend
|
||||||
|
treating all ten lanes as required when the caller claims to consume the
|
||||||
|
complete D&D workflow, while allowing another consumer to adopt a narrower
|
||||||
|
documented policy.
|
||||||
|
- Recommend retaining the receipt, the complete published bundle, and captured
|
||||||
|
diagnostic streams long enough to support provenance and failure analysis.
|
||||||
|
- Explain that `evidence-context.json` is a reading excerpt; authoritative
|
||||||
|
citations remain in lane payloads.
|
||||||
|
- Treat transcripts, lane artifacts, evidence context, manifests, and logs as
|
||||||
|
sensitive campaign data.
|
||||||
|
|
||||||
|
#### Compatibility Checklist
|
||||||
|
|
||||||
|
End with a concise checklist covering process exit, receipt schema, path
|
||||||
|
confinement, pipeline identity, index decoding, required descriptors,
|
||||||
|
descriptor schema/media compatibility, warnings and rejections, checksums or
|
||||||
|
retention, and secure handling. Compatibility should be based on published
|
||||||
|
receipt and artifact contracts rather than parsing a human version string.
|
||||||
|
|
||||||
|
### Update Existing Navigation
|
||||||
|
|
||||||
|
- Add a short link from `docs/consumers/subprocess.md` to the D&D-specific
|
||||||
|
workflow. Keep generic subprocess policy in the existing document.
|
||||||
|
- Add the guide to the documentation links in `README.md`.
|
||||||
|
- Extend the subprocess-consumer row in `docs/development.md` so maintainers
|
||||||
|
working on the D&D workflow are routed to the new guide and the canonical
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
### Verify Canonical Contract Documents
|
||||||
|
|
||||||
|
Review the linked integration documents and the complete example while writing
|
||||||
|
the guide. Correct an integration document only if repository inspection finds
|
||||||
|
an actual stale contract. Do not move schema definitions, field tables, CLI
|
||||||
|
flags, or configuration semantics into the new guide.
|
||||||
|
|
||||||
|
## Narratio Alignment
|
||||||
|
|
||||||
|
The guide may name Narratio as the motivating consumer and identify its current
|
||||||
|
final-trimmed transcript source. It must not claim that Narratio already has a
|
||||||
|
Notarius adapter or extraction stage. Until that feature is implemented,
|
||||||
|
Narratio-specific architecture, configuration, stage behavior, manifest
|
||||||
|
records, and artifact source IDs belong in Narratio's roadmap.
|
||||||
|
|
||||||
|
Once Narratio implements the integration, its own integration documentation
|
||||||
|
should link to this guide and the durable Notarius contracts instead of
|
||||||
|
repeating them.
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
|
||||||
|
Documentation implementation should include:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/notarius config validate \
|
||||||
|
--config examples/dnd-complete.config.yml \
|
||||||
|
--pipeline dnd-session
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Also verify all new and changed relative Markdown links, compare the artifact
|
||||||
|
inventory directly with the maintained complete configuration, and confirm
|
||||||
|
that commands and path semantics match the CLI and configuration references.
|
||||||
|
If the repository still has no automated link checker, record that fact and
|
||||||
|
perform a focused manual link review.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- A subprocess integrator can follow one D&D-focused guide from a Seriatim
|
||||||
|
transcript through safe discovery of every artifact configured by the
|
||||||
|
complete example.
|
||||||
|
- The guide makes stdout, stderr, exit-status, receipt, and path-confinement
|
||||||
|
responsibilities unambiguous.
|
||||||
|
- The ten configured D&D lanes and both configured pipeline-wide artifacts are
|
||||||
|
listed and linked to their canonical contracts.
|
||||||
|
- The guide distinguishes process success from the caller's required-artifact
|
||||||
|
policy.
|
||||||
|
- The profile-path and reference-path resolution rules are clearly stated.
|
||||||
|
- Existing navigation makes the guide discoverable.
|
||||||
|
- No volatile contract is defined in two places, and no unimplemented Narratio
|
||||||
|
behavior is presented as current.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- Implementing or documenting Narratio's future adapter or stage as current
|
||||||
|
Notarius behavior.
|
||||||
|
- Adding a new Notarius command, receipt version, output format, or artifact
|
||||||
|
schema.
|
||||||
|
- Duplicating the complete configuration or individual D&D payload schemas in
|
||||||
|
prose.
|
||||||
|
- Defining a universal partial-result policy for every Notarius consumer.
|
||||||
@@ -5,13 +5,207 @@ configuration, operations, internal, and integration docs. This roadmap records
|
|||||||
future work only. Items are ordered roughly by current value and specificity,
|
future work only. Items are ordered roughly by current value and specificity,
|
||||||
not as committed release dates.
|
not as committed release dates.
|
||||||
|
|
||||||
|
## Near-Term Validation And LLM Reliability
|
||||||
|
|
||||||
|
The following work forms one related program but should be promoted into
|
||||||
|
separate feature roadmaps and implemented in dependency order. PromptKit owns
|
||||||
|
structural output repair within one completion. Notarius owns stage candidates,
|
||||||
|
validator chains, semantic rejection policy, and whether another stage attempt
|
||||||
|
is warranted.
|
||||||
|
|
||||||
|
### 1. Upgrade To PromptKit v0.8.0
|
||||||
|
|
||||||
|
This item has been promoted to the standalone
|
||||||
|
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md) roadmap. That document owns the
|
||||||
|
release-by-release compatibility review, adopted features, structured-repair
|
||||||
|
policy, target integration boundary, acceptance criteria, and settled design
|
||||||
|
decisions.
|
||||||
|
|
||||||
|
### 2. Feedback-Aware Stage Validation Retries
|
||||||
|
|
||||||
|
- Model Notarius's corrective stage-retry conversation explicitly after
|
||||||
|
PromptKit v0.8.0. The first attempt sends the ordinary complete initial
|
||||||
|
prompt. If application validation rejects the resulting LLM-produced
|
||||||
|
candidate and another stage attempt is available, reconstruct that complete
|
||||||
|
initial prompt byte-for-byte and append exactly two messages: an assistant
|
||||||
|
message containing the defective response and an application-owned user
|
||||||
|
message detailing every applicable semantic validation error and requesting
|
||||||
|
one corrected, complete replacement response. This is a freshly constructed
|
||||||
|
correction request, not continuation of an accumulating conversation.
|
||||||
|
- Use the configured stage `retries` value as the one outer retry budget for
|
||||||
|
this loop. `retries: N` continues to mean at most `N` additional complete
|
||||||
|
chunk, extract, merge, or normalize attempts after the initial attempt,
|
||||||
|
whether an attempt is needed because of a producer error or semantic
|
||||||
|
rejection. Do not add a second semantic-correction count. PromptKit's
|
||||||
|
prompt-level `repair_attempts` budget is independent and internal to each
|
||||||
|
individual LLM completion, and does not consume or replenish the Notarius
|
||||||
|
stage budget.
|
||||||
|
- Extend the framework-managed validation boundary for chunk, extract, merge,
|
||||||
|
and normalize stages so a rejected LLM-produced candidate and its exact raw
|
||||||
|
model response remain available to construct the next stage attempt.
|
||||||
|
Deterministic producers cannot improve by repeating the same inputs; a
|
||||||
|
rejection from a deterministic stage is therefore terminal under the
|
||||||
|
configured rejection policy rather than consuming retries mechanically.
|
||||||
|
- Preserve the original session ID, selected profile, structured-output
|
||||||
|
contract, prompt inputs, and reusable prompt prefix. Carry only the latest
|
||||||
|
candidate and latest aggregate feedback; do not build an unbounded retry
|
||||||
|
conversation. Keep model-facing corrective guidance separate from
|
||||||
|
operator-facing diagnostics, and apply explicit size, redaction, and debug
|
||||||
|
disclosure rules to both.
|
||||||
|
- Run every applicable validator in the configured chain before deciding
|
||||||
|
whether to retry. Do not short-circuit merely because an earlier validator
|
||||||
|
rejected the candidate. Aggregate all semantic rejection reason codes and
|
||||||
|
corrective guidance into the retry message so one retry can address the
|
||||||
|
whole candidate. A validator is applicable only when its declared target and
|
||||||
|
prerequisites can be satisfied; record a deterministic skipped diagnostic
|
||||||
|
rather than invoking a validator on an input it cannot interpret. Initially
|
||||||
|
execute the chain sequentially in configured order so results, diagnostics,
|
||||||
|
costs, and feedback ordering remain deterministic; consider validator
|
||||||
|
concurrency only in response to measured latency.
|
||||||
|
- Continue running independent applicable validators after one validator
|
||||||
|
execution failure so the attempt retains as much useful diagnostic
|
||||||
|
information as practical. Do not present validator operational failures as
|
||||||
|
defects in the producer candidate and do not include them in corrective
|
||||||
|
feedback.
|
||||||
|
- Distinguish three terminal conditions and make their policies configurable
|
||||||
|
at a coherent pipeline or binding scope:
|
||||||
|
- **producer structural failure:** PromptKit could not return a usable
|
||||||
|
structured candidate after its repair budget. Default to `fail_run`; an
|
||||||
|
allowed alternative may record a terminal stage or lane rejection where
|
||||||
|
execution can safely continue, but may not accept the invalid output;
|
||||||
|
- **semantic rejection:** one or more validators completed and rejected the
|
||||||
|
candidate. Default to `fail_run` after corrective stage retries are
|
||||||
|
exhausted; allow an explicit alternative that records the existing
|
||||||
|
rejected-output outcome without advancing that output;
|
||||||
|
- **validator execution failure:** a validator could not produce a valid
|
||||||
|
decision because of generation, structural-output, transport, or internal
|
||||||
|
failure. Default to a genuine warning and an explicitly recorded
|
||||||
|
`validation_incomplete` or equivalent degraded state while allowing the
|
||||||
|
candidate to continue; allow strict configuration to fail the run instead.
|
||||||
|
- An LLM-backed validator uses the same scheduled PromptKit boundary as every
|
||||||
|
other LLM-backed module. Its own response may use PromptKit's bounded
|
||||||
|
structural repair. Distinguish its possible output states:
|
||||||
|
- output rejected by PromptKit's structural contract should consume only the
|
||||||
|
validator prompt's configured PromptKit repair budget;
|
||||||
|
- output that is structurally valid but violates a deterministically
|
||||||
|
checkable validator-result invariant should be classified as a validator
|
||||||
|
execution failure;
|
||||||
|
- output that satisfies the complete validator-result contract is the
|
||||||
|
validator's decision, even though an LLM judgment may remain imperfect.
|
||||||
|
Automatically judging that judgment would require another semantic
|
||||||
|
validator and is outside this feature.
|
||||||
|
If the validator cannot return a contract-valid decision, do not recursively
|
||||||
|
create another Notarius semantic-validation loop around it. Apply the
|
||||||
|
configured validator-failure policy. The default warning must identify the
|
||||||
|
validator and affected stage without exposing sensitive content.
|
||||||
|
- Separate validator execution retry from producer correction. A transient
|
||||||
|
validator operational failure must not automatically discard and regenerate
|
||||||
|
an otherwise usable producer candidate. Any bounded retry of the validator
|
||||||
|
itself should reuse that same immutable candidate and remain subordinate to
|
||||||
|
PromptKit and provider retry behavior.
|
||||||
|
- Preserve attempt-level provenance, cumulative token usage, validator
|
||||||
|
outcomes, aggregated correction feedback, and terminal policy decisions in
|
||||||
|
the debug and manifest models without copying raw source material into
|
||||||
|
ordinary errors or durable summaries.
|
||||||
|
- Define terminal-outcome precedence. A semantic rejection dominates a
|
||||||
|
validator execution failure for the same candidate: use the completed
|
||||||
|
rejections to correct the producer while separately recording incomplete
|
||||||
|
validation. If a later candidate has no semantic rejection but one validator
|
||||||
|
still fails, apply the configured validator-failure policy to that candidate.
|
||||||
|
Never allow a known semantic rejection to become accepted through a
|
||||||
|
warn-and-continue setting, and never accept a structurally invalid producer
|
||||||
|
response. Permissive policy may preserve a rejected-output outcome or accept
|
||||||
|
a structurally valid candidate with explicitly incomplete validation; it may
|
||||||
|
not relabel known-invalid output as approved.
|
||||||
|
|
||||||
|
Before implementation, record the generic validation and retry state machine
|
||||||
|
in an ADR. The ADR should own the separation between PromptKit repair and
|
||||||
|
Notarius correction, use of the existing stage-retry budget, reconstruction of
|
||||||
|
correction conversations, all-applicable-validator aggregation, deterministic
|
||||||
|
validator ordering, non-recursive validator failure handling, outcome
|
||||||
|
precedence, default fail-open/fail-closed choices, configurable terminal
|
||||||
|
policies, and provenance and sensitive-data constraints. A dependency-upgrade
|
||||||
|
ADR is not needed for PromptKit v0.8.0 itself. Current behavior remains
|
||||||
|
authoritative until the validation ADR is implemented and the canonical
|
||||||
|
architecture, configuration, operations, and internal documentation are
|
||||||
|
updated.
|
||||||
|
|
||||||
|
### 3. D&D Combat Scene Semantic Validation
|
||||||
|
|
||||||
|
- Add an optional production LLM-backed D&D validator that determines whether
|
||||||
|
proposed scene boundaries and classifications represent substantive active
|
||||||
|
combat correctly. Its central quality goal is that active combat is kept in
|
||||||
|
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||||
|
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||||
|
- Resolve the validator's exact target before implementation. The current
|
||||||
|
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||||
|
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||||
|
`recap`, and `meta` classification. The preferred initial placement is
|
||||||
|
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||||
|
can compare one proposed kind with the corresponding transcript chunk.
|
||||||
|
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||||
|
question that can be answered from the complete transcript and proposed
|
||||||
|
range map, such as whether one continuous combat was fragmented across
|
||||||
|
inappropriate scene boundaries. Do not duplicate the same classification
|
||||||
|
judgment at both stages. Moving classification into chunk-plan annotations
|
||||||
|
would change the deliberately minimal, annotation-free chunk contract and
|
||||||
|
requires an explicit architecture review before it is selected.
|
||||||
|
- Validate both false negatives and false positives: a non-combat kind must not
|
||||||
|
omit substantive active combat, and a combat kind must be supported by such
|
||||||
|
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||||
|
extraction runs only for an exact `combat` scene classification; semantic
|
||||||
|
review improves the upstream classification but does not replace that gate.
|
||||||
|
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||||
|
structured response schema, and let PromptKit repair structural validator
|
||||||
|
output within its bounded budget. A contract-invalid final validator response
|
||||||
|
is a validator execution failure, not a semantic rejection and not a reason
|
||||||
|
to recursively validate the validator.
|
||||||
|
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||||
|
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||||
|
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||||
|
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||||
|
added calls, latency, and token cost before placing it in the production
|
||||||
|
default chain.
|
||||||
|
- An ADR is not required if classification remains owned by
|
||||||
|
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||||
|
Create or supersede an ADR if the work transfers scene classification into
|
||||||
|
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||||
|
contract.
|
||||||
|
|
||||||
|
### 4. Warning Signal And Presentation Reform
|
||||||
|
|
||||||
|
- Audit every warning producer and representative successful runs. Ordinary
|
||||||
|
success producing dozens of warnings is a failed operator experience: the
|
||||||
|
volume obscures actionable problems and trains operators to ignore the
|
||||||
|
warning channel.
|
||||||
|
- Define a small warning taxonomy that distinguishes actionable degradation,
|
||||||
|
incomplete validation, lossy fallback, and data-quality risk from routine
|
||||||
|
normalization observations or informational diagnostics. Preserve detailed
|
||||||
|
traceability in debug or manifest data without promoting every observation
|
||||||
|
to a top-level CLI warning.
|
||||||
|
- Consider stable deduplication and aggregation by scope and reason code,
|
||||||
|
bounded samples plus omitted counts, and a concise CLI summary with a path to
|
||||||
|
detailed diagnostics. Do not suppress genuine validator execution failures
|
||||||
|
merely to reduce the count.
|
||||||
|
- Decide which warnings affect process status, rejection summaries, durable run
|
||||||
|
receipts, or only debug output. Ensure warning ordering and aggregation are
|
||||||
|
deterministic across concurrent execution.
|
||||||
|
- Establish a representative warning-volume acceptance target and human review
|
||||||
|
workflow before changing individual producers piecemeal. The intended result
|
||||||
|
is not zero warnings; it is a small set in which every surfaced warning merits
|
||||||
|
operator attention.
|
||||||
|
- This work does not require an ADR unless it changes validation acceptance,
|
||||||
|
failure, or durable contract semantics. CLI presentation and diagnostic
|
||||||
|
taxonomy otherwise belong in a feature roadmap followed by updates to their
|
||||||
|
canonical configuration, operations, integration, and internal documents.
|
||||||
|
|
||||||
## Near-Term D&D Pipeline
|
## Near-Term D&D Pipeline
|
||||||
|
|
||||||
### Evaluate Spell Extraction And Normalization
|
### Evaluate Spell Extraction And Normalization
|
||||||
|
|
||||||
- Evaluate ordinary extraction retries and the completed normalization path
|
- Evaluate ordinary extraction retries and the completed normalization path
|
||||||
against a human-reviewed transcript set before adding repair-aware retries or
|
against a human-reviewed transcript set before and after adopting the shared
|
||||||
an LLM-backed semantic validator.
|
PromptKit repair and Notarius validation-retry policies above.
|
||||||
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
||||||
validator, and normalizer development. Treat model-quality review as an
|
validator, and normalizer development. Treat model-quality review as an
|
||||||
iterative human evaluation aid, not a deterministic correctness gate.
|
iterative human evaluation aid, not a deterministic correctness gate.
|
||||||
@@ -24,28 +218,51 @@ not as committed release dates.
|
|||||||
|
|
||||||
## Shared Normalization And Quality Work
|
## Shared Normalization And Quality Work
|
||||||
|
|
||||||
### Generic LLM-Assisted Deduplication
|
The implemented source-backed core and initial D&D registry adoption are
|
||||||
|
described by [Module Internals](../internal/modules.md#semantic-reconciliation)
|
||||||
|
and
|
||||||
|
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||||
|
The sections below keep broader extensions deferred.
|
||||||
|
|
||||||
- Add a reusable normalizer that asks an LLM to identify duplicate sets in a
|
### Large-Collection Semantic Reconciliation
|
||||||
list and propose one replacement element for each set.
|
|
||||||
- Define the minimum domain-neutral input contract, initially an ordered list
|
|
||||||
whose elements retain stable unique IDs as internal deterministic state.
|
|
||||||
Model proposals use contextual descriptors, or a specifically justified
|
|
||||||
request-local short label, rather than durable IDs. Artifact-kind
|
|
||||||
registrations or adapters may expose that structure without moving domain
|
|
||||||
rules into the generic package.
|
|
||||||
- Keep mutation deterministic: parse and validate the model's duplicate groups,
|
|
||||||
resolve every supplied descriptor or local label exactly, reject overlapping
|
|
||||||
or malformed groups, prevent unrelated insertion or deletion, and apply only
|
|
||||||
approved replacement operations in code.
|
|
||||||
- Preserve provenance needed for audit and downstream validation, and emit
|
|
||||||
warnings describing every collapsed group.
|
|
||||||
- Evaluate batching and context-window limits before applying the normalizer to
|
|
||||||
large artifact collections.
|
|
||||||
|
|
||||||
The model may use its own domain knowledge to judge semantic duplication; the
|
- Evaluate deterministic candidate blocking only after representative registry
|
||||||
generic implementation is responsible only for the common proposal contract,
|
inputs exceed the active roadmap's bounded single-request limits. Blocking
|
||||||
safety checks, and deterministic application of accepted changes.
|
should use cheap, explainable signals to form plausible comparison sets while
|
||||||
|
preserving the possibility that a duplicate appears outside a lexical name
|
||||||
|
match.
|
||||||
|
- Define correctness for candidates that appear in more than one block,
|
||||||
|
conflicting canonical selections, transitive identity across blocks, retry
|
||||||
|
isolation, and deterministic final ordering before implementation.
|
||||||
|
- Prefer a reconciliation graph or union plan with explicit conflict checks
|
||||||
|
over arbitrary fixed-size slices. Never silently treat a batch boundary as
|
||||||
|
evidence that two candidates are distinct.
|
||||||
|
- Record per-request bounds, block provenance, model calls, discarded
|
||||||
|
proposals, and final group derivation well enough to audit a collapse.
|
||||||
|
|
||||||
|
### Operator-Selected Semantic Policies
|
||||||
|
|
||||||
|
- Consider allowing an operator to select an approved semantic-policy prompt
|
||||||
|
for a typed reconciliation module without replacing the shared protocol,
|
||||||
|
response schema, or deterministic safety rules.
|
||||||
|
- Define the trusted asset source, configuration syntax, compatibility checks,
|
||||||
|
startup validation, provenance, prompt fingerprinting, checkpoint effects,
|
||||||
|
and support boundary before exposing the option.
|
||||||
|
- Prefer selection among registered, typed-policy-compatible prompt assets over
|
||||||
|
arbitrary filesystem prompt paths. Do not add this flexibility until an
|
||||||
|
operator workflow requires it; artifact-family-owned policy remains simpler
|
||||||
|
and safer for the initial implementation.
|
||||||
|
|
||||||
|
### Broader Reconciliation Inputs And Module Selection
|
||||||
|
|
||||||
|
- Revisit alternate context providers when a concrete non-source-backed entity
|
||||||
|
collection needs semantic reconciliation. Any extension must preserve the
|
||||||
|
same request-local identity, deterministic proposal validation, provenance,
|
||||||
|
and typed application guarantees.
|
||||||
|
- Consider a selectable generic normalizer only if Notarius gains a real
|
||||||
|
domain-neutral typed artifact contract that can safely support it. Do not
|
||||||
|
weaken exact artifact registration or introduce reflection-based arbitrary
|
||||||
|
JSON mutation merely to expose a universal module key.
|
||||||
|
|
||||||
### Validation And Review
|
### Validation And Review
|
||||||
|
|
||||||
@@ -102,6 +319,18 @@ checkpoint reuse, when an older artifact may be decoded or adapted, and when a
|
|||||||
producer or all dependents must be recomputed. Do not add a general migration
|
producer or all dependents must be recomputed. Do not add a general migration
|
||||||
framework until an actual contract change requires one.
|
framework until an actual contract change requires one.
|
||||||
|
|
||||||
|
### Artifact-family-oriented physical packaging
|
||||||
|
|
||||||
|
[ADR-0004](../adr/0004-package-modules-by-domain.md) currently groups production
|
||||||
|
extensions by domain and then by pipeline stage. After artifact-family
|
||||||
|
ownership terminology is established and more families span extraction,
|
||||||
|
normalization, validation, codecs, references, and assets, reassess whether a
|
||||||
|
feature-first physical layout would improve navigation and reduce scattered
|
||||||
|
changes enough to justify a repository-wide package migration. Any change must
|
||||||
|
address Go dependency cycles, registrar ownership, stable public module keys,
|
||||||
|
and supersession of the affected ADR-0004 decision. Conceptual artifact-family
|
||||||
|
ownership does not by itself require this move.
|
||||||
|
|
||||||
## Blue-Sky Platform And Operations
|
## Blue-Sky Platform And Operations
|
||||||
|
|
||||||
These ideas are intentionally less specified. Promote one into an earlier
|
These ideas are intentionally less specified. Promote one into an earlier
|
||||||
@@ -117,7 +346,6 @@ section only after a concrete workflow, contract, and priority emerge.
|
|||||||
### Distribution And Operations
|
### Distribution And Operations
|
||||||
|
|
||||||
- Packaged release artifacts for alpha distribution.
|
- Packaged release artifacts for alpha distribution.
|
||||||
- A documented versioning and release process.
|
|
||||||
- Optional generated example-output fixtures with a regeneration procedure.
|
- Optional generated example-output fixtures with a regeneration procedure.
|
||||||
- Additional diagnostics or reporting views.
|
- Additional diagnostics or reporting views.
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
520
docs/roadmap/promptkit-v0.8.md
Normal file
520
docs/roadmap/promptkit-v0.8.md
Normal file
@@ -0,0 +1,520 @@
|
|||||||
|
# PromptKit v0.8.0 Upgrade
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Proposed.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Upgrade Notarius from PromptKit v0.5.0 to v0.8.0 and deliberately adopt the
|
||||||
|
useful correctness, profile-composition, provider-diagnostic, backend, and
|
||||||
|
structured-output-repair capabilities introduced in PromptKit v0.6.0, v0.7.0,
|
||||||
|
and v0.8.0.
|
||||||
|
|
||||||
|
The upgrade should improve structured-output reliability without confusing
|
||||||
|
PromptKit's bounded deterministic repair with Notarius's existing stage retry
|
||||||
|
budget or the future feedback-aware semantic-validation loop. PromptKit types
|
||||||
|
and provider behavior must remain behind Notarius's transport-neutral LLM
|
||||||
|
boundary.
|
||||||
|
|
||||||
|
## Current State
|
||||||
|
|
||||||
|
Notarius currently pins PromptKit v0.5.0. Its production adapter prepares one
|
||||||
|
frozen execution, records credential-redacted details, and runs that same
|
||||||
|
prepared value. It maps PromptKit capacity failures to an application-owned
|
||||||
|
error, maps failed structured validation to `ErrInvalidStructuredOutput`, and
|
||||||
|
returns PromptKit's raw validated bytes and usage metadata.
|
||||||
|
|
||||||
|
Every maintained production prompt uses JSON Schema validation and currently
|
||||||
|
declares `repair_attempts: 0`. Notarius stage bindings separately expose
|
||||||
|
`retries`, which reruns a complete stage operation after an error or rejected
|
||||||
|
candidate. The two mechanisms have different ownership and must remain
|
||||||
|
independent.
|
||||||
|
|
||||||
|
Notarius also maintains:
|
||||||
|
|
||||||
|
- embedded prompt, schema, and fallback-profile filesystems;
|
||||||
|
- operator profile-file and profile-directory sources;
|
||||||
|
- one optional conventional `local` backend registration;
|
||||||
|
- explicit profile preflight through PromptKit inspection;
|
||||||
|
- one application-wide scheduled LLM client around the PromptKit adapter;
|
||||||
|
- PromptKit profile-source fingerprints for checkpoint safety; and
|
||||||
|
- redacted debug and manifest provenance at application-owned boundaries.
|
||||||
|
|
||||||
|
The upgrade must preserve those established responsibilities while revising
|
||||||
|
the pinned integration contract and any behavior affected by the three
|
||||||
|
intervening releases.
|
||||||
|
|
||||||
|
This roadmap is based on PromptKit's pinned release guides for
|
||||||
|
[v0.6.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.6.0.md),
|
||||||
|
[v0.7.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.7.0.md),
|
||||||
|
and
|
||||||
|
[v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.8.0.md),
|
||||||
|
plus the public API and format documentation at the v0.8.0 tag.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
- `go.mod` and `go.sum` pin PromptKit v0.8.0 without a local replacement or
|
||||||
|
vendored copy.
|
||||||
|
- Every maintained PromptKit prompt and profile prepares successfully under
|
||||||
|
v0.8.0's stricter validation and source-loading rules.
|
||||||
|
- Eligible Notarius structured completions use one PromptKit corrective call by
|
||||||
|
default after a structurally invalid response. Operators can explicitly set
|
||||||
|
a value from zero through three for a configured pipeline, with a more local
|
||||||
|
LLM-backed binding override where needed.
|
||||||
|
- PromptKit repair remains an inner operation within one Notarius stage
|
||||||
|
attempt. It never consumes or replenishes the binding's `retries` budget.
|
||||||
|
- A successful repaired result exposes cumulative usage and the actual repair
|
||||||
|
count to Notarius's application-owned response and debug models. A repaired
|
||||||
|
success is not itself a warning.
|
||||||
|
- Exhausted PromptKit validation remains an invalid structured-output result,
|
||||||
|
preserving the final candidate and diagnostics for debug and for any
|
||||||
|
applicable outer Notarius stage policy. Invalid structured output is never
|
||||||
|
accepted merely because the repair budget was exhausted.
|
||||||
|
- Profile inheritance, the built-in Rakestrawhome backend/profile, optional
|
||||||
|
credential behavior, and structured generation errors work through the
|
||||||
|
existing Notarius PromptKit boundary and are accurately documented.
|
||||||
|
- Provider-specific PromptKit types do not escape `internal/framework/llm`.
|
||||||
|
- Checkpoint identity, effective configuration, redacted summaries, and debug
|
||||||
|
provenance reflect every execution-affecting repair or profile change.
|
||||||
|
- Current documentation pins and describes v0.8.0; future Notarius semantic
|
||||||
|
validation retries remain roadmap behavior rather than being conflated with
|
||||||
|
this dependency upgrade.
|
||||||
|
|
||||||
|
## Release-by-Release Adoption
|
||||||
|
|
||||||
|
### PromptKit v0.6.0: Correctness, Safety, And Efficiency
|
||||||
|
|
||||||
|
PromptKit v0.6.0 adds no public declarations, but intentionally rejects several
|
||||||
|
formerly permissive or ambiguous inputs. The upgrade must audit Notarius's
|
||||||
|
embedded and operator-facing integration against these rules:
|
||||||
|
|
||||||
|
- YAML `id` and `version` metadata, rather than filenames, define prompt and
|
||||||
|
profile identity.
|
||||||
|
- Prompt `content_file` paths are exact, relative, contained paths; built-in
|
||||||
|
file artifacts must resolve to regular files.
|
||||||
|
- execution controls, output contracts, and repair budgets must be finite and
|
||||||
|
within their documented ranges;
|
||||||
|
- provider endpoints must be absolute HTTP or HTTPS URLs with a host and no
|
||||||
|
user information, query, or fragment;
|
||||||
|
- JSON documents and successful provider responses contain exactly one value;
|
||||||
|
- successful provider responses are bounded to 16 MiB; and
|
||||||
|
- JSON-compatible values are bounded for depth and expansion.
|
||||||
|
|
||||||
|
Notarius should rely on PromptKit for these rules rather than duplicate its
|
||||||
|
parsers or internal limits. Existing Notarius validation may retain a narrower
|
||||||
|
application rule where it has independent value, but overlapping validation
|
||||||
|
must agree with PromptKit and must not accept a value PromptKit will reject
|
||||||
|
later.
|
||||||
|
|
||||||
|
The upgrade automatically receives operation-local schema-plan reuse,
|
||||||
|
artifact-text memoization, improved cancellation checks, and transport error
|
||||||
|
identity preservation. Notarius should verify these changes through its real
|
||||||
|
adapter boundary and avoid adding a second cache or response-body layer that
|
||||||
|
would duplicate PromptKit's ownership.
|
||||||
|
|
||||||
|
### PromptKit v0.7.0: Profiles, Backend Access, And Generation Errors
|
||||||
|
|
||||||
|
#### Profile Inheritance
|
||||||
|
|
||||||
|
Operator profiles may use `base_profile` to alias or selectively refine a
|
||||||
|
built-in, fallback, or higher-precedence operator profile. Notarius must pass
|
||||||
|
profile sources through unchanged and let PromptKit own parent lookup, merge
|
||||||
|
rules, source precedence, cycle detection, and fully resolved prepared targets.
|
||||||
|
|
||||||
|
Preflight inspection must resolve inherited profiles through the same source
|
||||||
|
and backend composition used at execution. The selected leaf profile ID remains
|
||||||
|
the public profile identity, while effective backend, endpoint, model, and
|
||||||
|
reasoning provenance reflect the resolved chain. Notarius must not implement a
|
||||||
|
second inheritance parser.
|
||||||
|
|
||||||
|
The existing complete `dnd-extraction` fallback remains a standalone profile:
|
||||||
|
PromptKit v0.8.0 does not provide a built-in `openai/gpt-5.6-luna` profile that
|
||||||
|
would be an appropriate parent. Documentation should nevertheless explain how
|
||||||
|
operators can use inheritance for environment-specific workload profiles and
|
||||||
|
should link to PromptKit's pinned format contract rather than duplicate its
|
||||||
|
field-by-field merge algorithm.
|
||||||
|
|
||||||
|
Checkpoint safety must cover inherited behavior. Operator file/directory
|
||||||
|
digests already cover changes to definitions in those sources, fallback asset
|
||||||
|
digests cover application parents, and the PromptKit built-in catalog marker
|
||||||
|
must change from its v0.5.0 identity to v0.8.0 so a changed built-in parent
|
||||||
|
cannot reuse an incompatible checkpoint.
|
||||||
|
|
||||||
|
#### Rakestrawhome Backend And Profile
|
||||||
|
|
||||||
|
PromptKit's reserved `rakestrawhome` backend and
|
||||||
|
`rakestrawhome-gemma-4-31b` profile become available without Notarius-specific
|
||||||
|
registration. Notarius must not register or shadow the reserved backend ID.
|
||||||
|
Profile preflight, backend-capacity reporting, scheduling, generation, and
|
||||||
|
provenance should work for it through the same generic paths used by OpenRouter
|
||||||
|
and `local`.
|
||||||
|
|
||||||
|
The D&D default remains `dnd-extraction`; this upgrade does not silently move a
|
||||||
|
production workload to Rakestrawhome. Operator documentation should identify
|
||||||
|
the built-in profile as an available selection and link to PromptKit for its
|
||||||
|
endpoint, credential environment, model, and capacity defaults.
|
||||||
|
|
||||||
|
#### Optional Credentials
|
||||||
|
|
||||||
|
An absent or blank optional `APIKeyEnv` now causes PromptKit to omit the
|
||||||
|
`Authorization` header and send the request. Notarius must not restore the old
|
||||||
|
failure behavior by pre-reading provider credential environment variables or
|
||||||
|
by adding provider-specific authentication logic.
|
||||||
|
|
||||||
|
Profile inspection may report an explicit `APIKeyRequired` policy without
|
||||||
|
reading the credential, and execution remains the boundary at which that
|
||||||
|
requirement is enforced. For optional profiles, an authentication-requiring
|
||||||
|
provider may instead return a structured 401 or 403 generation failure. The
|
||||||
|
configuration and operations documentation must explain this distinction.
|
||||||
|
Notarius does not currently expose PromptKit's in-memory profile-registration
|
||||||
|
API to operators, and PromptKit's filesystem profile format does not expose
|
||||||
|
`APIKeyRequired`; therefore Notarius must not promise that an operator profile
|
||||||
|
can force local credential preflight. Operators should provision the named
|
||||||
|
environment variable, while Notarius should preserve the provider's structured
|
||||||
|
authentication failure when it is absent.
|
||||||
|
|
||||||
|
Notarius must continue to document mechanisms and environment-variable names,
|
||||||
|
never secret values.
|
||||||
|
|
||||||
|
#### Structured Generation Errors
|
||||||
|
|
||||||
|
The adapter should recognize `*promptkit.GenerationError` with `errors.As` and
|
||||||
|
translate useful information into an immutable, provider-neutral Notarius
|
||||||
|
error classification. At minimum, retain the HTTP status code so callers and
|
||||||
|
future retry policy can distinguish transport success with provider rejection
|
||||||
|
from other generation failures.
|
||||||
|
|
||||||
|
PromptKit's provider code, type, and message accessors are bounded but remain
|
||||||
|
untrusted and potentially sensitive. They must never appear automatically in
|
||||||
|
ordinary CLI output, warnings, manifests, checkpoint identity, or cache data.
|
||||||
|
If retained for an explicitly requested debug trace, they must pass through
|
||||||
|
Notarius's known-secret and bearer redaction and remain clearly identified as
|
||||||
|
untrusted provider diagnostics. Default error formatting should continue to
|
||||||
|
use a bounded, redacted application-owned message.
|
||||||
|
|
||||||
|
Capacity and cancellation retain their current more specific classifications
|
||||||
|
and precedence. This upgrade does not add automatic provider-error retry
|
||||||
|
classification; it only preserves safe structured data needed for diagnosis
|
||||||
|
and later policy.
|
||||||
|
|
||||||
|
### PromptKit v0.8.0: Bounded Structured-Output Repair
|
||||||
|
|
||||||
|
#### Default Policy
|
||||||
|
|
||||||
|
Every maintained production prompt whose output is consumed as structured data
|
||||||
|
should declare one repair attempt. All current production prompts use eligible
|
||||||
|
JSON Schema validation, so no current prompt needs a zero default merely
|
||||||
|
because of its output mode.
|
||||||
|
|
||||||
|
One repair means at most one corrective generation after the initial
|
||||||
|
candidate. PromptKit reconstructs the immutable original conversation and
|
||||||
|
appends only the latest invalid assistant candidate and latest deterministic
|
||||||
|
validation diagnostics. It preserves the selected target, direct session ID,
|
||||||
|
provider-native structured-output contract, and backend capacity policy. This
|
||||||
|
shape preserves the original cacheable prompt prefix and avoids accumulating
|
||||||
|
unbounded failed history.
|
||||||
|
|
||||||
|
The default is deliberately small. A single repair captures the common case in
|
||||||
|
which a capable model can correct malformed JSON or a schema violation after
|
||||||
|
receiving an exact diagnostic, while bounding the extra latency and cost of a
|
||||||
|
single structured completion.
|
||||||
|
|
||||||
|
#### Configuration Contract
|
||||||
|
|
||||||
|
The public configuration is an optional, presence-aware
|
||||||
|
`structured_output_repair_attempts` integer at pipeline scope and at each
|
||||||
|
LLM-backed module or validator binding. Its effective precedence is:
|
||||||
|
|
||||||
|
1. the binding value, when present;
|
||||||
|
2. the pipeline value, when present; and
|
||||||
|
3. the selected prompt's declared `repair_attempts` value.
|
||||||
|
|
||||||
|
The value must be from zero through three. Explicit zero disables PromptKit
|
||||||
|
repair at that scope. A deterministic binding must reject the field because it
|
||||||
|
cannot perform structured LLM repair. Validator bindings may use it only when
|
||||||
|
the selected validator is LLM-backed. Shorthand module bindings continue to
|
||||||
|
inherit the pipeline or prompt default.
|
||||||
|
|
||||||
|
The long, provider-neutral name is intentional: it distinguishes PromptKit's
|
||||||
|
inner structural repair from the existing binding `retries` field, which owns
|
||||||
|
complete stage attempts, without exposing a dependency name in generic
|
||||||
|
pipeline contracts.
|
||||||
|
|
||||||
|
The effective value must survive file parsing, cloning, redacted summaries,
|
||||||
|
pipeline resolution, and pipeline digest construction without pointer aliasing
|
||||||
|
or loss of presence. It must affect checkpoint identity because it can change
|
||||||
|
the selected result, latency, token usage, and provider cost.
|
||||||
|
|
||||||
|
#### Adapter Contract
|
||||||
|
|
||||||
|
The transport-neutral structured-completion request should carry an optional
|
||||||
|
application-owned structural-repair budget. No `promptkit.OutputContract` or
|
||||||
|
other PromptKit type may cross the adapter boundary.
|
||||||
|
|
||||||
|
PromptKit v0.8.0 request validation replaces the complete prompt output
|
||||||
|
contract rather than merging one field. When Notarius has a configured
|
||||||
|
override, the adapter must therefore inspect the selected prompt, copy its
|
||||||
|
normalized declared format, validation mode, and schema path, change only the
|
||||||
|
repair count, and supply that complete contract on the prepared request. A nil
|
||||||
|
override continues to use the prompt declaration directly. Inspection and
|
||||||
|
preparation must use the same immutable engine sources; a small adapter-local
|
||||||
|
cache keyed by normalized prompt ID and version is acceptable but not required
|
||||||
|
without measured need.
|
||||||
|
|
||||||
|
This approach prevents configuration from accidentally dropping JSON Schema
|
||||||
|
validation, avoids duplicating schema paths in pipeline YAML, and keeps prompt
|
||||||
|
assets authoritative for every output-contract field other than the explicit
|
||||||
|
operator override.
|
||||||
|
|
||||||
|
The transport-neutral structured-completion response should report the actual
|
||||||
|
number of PromptKit repair calls. PromptKit's returned token usage is already
|
||||||
|
cumulative and must be passed through without re-summing it. Debug records
|
||||||
|
should distinguish the configured budget from the actual count. Ordinary run
|
||||||
|
manifests need not gain raw prompt or response data merely to report repairs;
|
||||||
|
any durable aggregate should be added only if it has a clear consumer contract.
|
||||||
|
|
||||||
|
#### Result And Failure Semantics
|
||||||
|
|
||||||
|
- A valid initial candidate returns normally with zero actual repairs.
|
||||||
|
- A valid corrected candidate returns normally with cumulative usage and its
|
||||||
|
positive actual repair count. It does not emit a warning solely because a
|
||||||
|
repair occurred.
|
||||||
|
- Exhausting the repair budget returns PromptKit's final candidate and failed
|
||||||
|
validation result. The adapter maps this to
|
||||||
|
`ErrInvalidStructuredOutput`, preserves the response and debug material, and
|
||||||
|
does not decode or accept the candidate.
|
||||||
|
- An explicitly empty or whitespace-only candidate participates in the
|
||||||
|
declared structural validation and repair flow. Missing, `null`, or
|
||||||
|
non-string provider content remains a malformed provider response.
|
||||||
|
- A generation failure during a corrective call is an operational generation
|
||||||
|
failure and uses the same safe structured-error adaptation as an initial
|
||||||
|
generation failure.
|
||||||
|
- Context cancellation remains authoritative throughout the initial and
|
||||||
|
corrective calls.
|
||||||
|
|
||||||
|
PromptKit repair happens inside one scheduled `CompleteStructured` operation.
|
||||||
|
The Notarius scheduler holds one permit for that logical operation while
|
||||||
|
PromptKit performs its initial and serial corrective calls; PromptKit
|
||||||
|
reacquires its own selected-backend capacity for each corrective generation.
|
||||||
|
Because corrective calls are serial, this cannot expand actual concurrent
|
||||||
|
provider work beyond the number of admitted Notarius operations, but
|
||||||
|
documentation must stop describing the Notarius permit as a separate admission
|
||||||
|
event for every internal repair call.
|
||||||
|
|
||||||
|
One `CompleteStructured` invocation with effective PromptKit repair budget `R`
|
||||||
|
may make at most `R + 1` provider calls. If one stage attempt makes `C`
|
||||||
|
structured-completion invocations, a binding with `retries: N` has an upper
|
||||||
|
bound of `(N + 1) * C * (R + 1)` provider calls; `C` may itself be a bounded,
|
||||||
|
data-dependent module property, as it is for batched semantic reconciliation.
|
||||||
|
LLM-backed validators have their own corresponding invocation counts, budgets,
|
||||||
|
and costs. These formulas are upper bounds, not promises that every failure is
|
||||||
|
retryable or that every attempt reaches the provider.
|
||||||
|
|
||||||
|
## Profile And Prompt Source Compatibility
|
||||||
|
|
||||||
|
The upgrade must preserve Notarius's source precedence: an operator source,
|
||||||
|
then registered application fallback profiles, then PromptKit built-ins. A
|
||||||
|
selected malformed definition remains authoritative and fails rather than
|
||||||
|
falling through. Parent resolution introduced by profile inheritance observes
|
||||||
|
that same precedence.
|
||||||
|
|
||||||
|
All embedded prompt manifests, shared content fragments, response schemas, and
|
||||||
|
fallback profiles must be prepared or inspected offline under v0.8.0. The
|
||||||
|
review should specifically catch:
|
||||||
|
|
||||||
|
- IDs inferred accidentally from filenames;
|
||||||
|
- stale or escaping `content_file` paths;
|
||||||
|
- missing or non-regular embedded artifacts;
|
||||||
|
- repair values outside zero through three or paired with ineligible
|
||||||
|
validation;
|
||||||
|
- schemas or examples that are not exact single JSON documents;
|
||||||
|
- unsupported endpoint forms; and
|
||||||
|
- JSON-compatible variables or profile extras that exceed upstream bounds.
|
||||||
|
|
||||||
|
No prompt prose, schema shape, durable D&D artifact contract, or default D&D
|
||||||
|
model should change merely to exercise the dependency. Prompt manifests should
|
||||||
|
change only as needed to enable the adopted repair default and satisfy v0.8.0
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
## Provenance, Debugging, And Security
|
||||||
|
|
||||||
|
- Update the opaque PromptKit built-in profile-catalog identity from v0.5.0 to
|
||||||
|
v0.8.0. Do not hash or publish PromptKit's internal catalog bytes.
|
||||||
|
- Ensure a prompt's repair default remains covered by its existing prompt asset
|
||||||
|
fingerprint and a configured effective override remains covered by the
|
||||||
|
resolved pipeline digest.
|
||||||
|
- Preserve selected leaf profile identity while recording the inherited
|
||||||
|
effective target already exposed by PromptKit inspection and prepared
|
||||||
|
details.
|
||||||
|
- Add actual structural-repair count and, when useful, the configured budget to
|
||||||
|
application-owned debug material. Token totals remain PromptKit's cumulative
|
||||||
|
values.
|
||||||
|
- Do not generate a warning for a successful repair. Repair exhaustion is an
|
||||||
|
invalid-output failure, while provider rejection is a generation failure.
|
||||||
|
- Never expose raw provider diagnostic fields without explicit debug capture
|
||||||
|
and application redaction. Do not place them in normal errors or durable
|
||||||
|
summaries.
|
||||||
|
- Preserve context and transport error identity sufficiently for
|
||||||
|
`errors.Is`-based cancellation and deadline handling after adapting the
|
||||||
|
external error.
|
||||||
|
|
||||||
|
## Documentation And Examples
|
||||||
|
|
||||||
|
Implementation should update current-state documentation only when the new
|
||||||
|
behavior lands:
|
||||||
|
|
||||||
|
- `docs/integrations/pkg-promptkit.md` must pin v0.8.0 and define the revised
|
||||||
|
prepared-execution, repair, profile-inheritance, backend, credential, and
|
||||||
|
error-adaptation boundary.
|
||||||
|
- `docs/config.md` must own the repair configuration fields, precedence,
|
||||||
|
allowed range, explicit-zero behavior, profile inheritance availability, and
|
||||||
|
optional credential semantics.
|
||||||
|
- `docs/operations.md` must explain structural repair cost, timeout and
|
||||||
|
concurrency effects, credential failures, and its distinction from stage
|
||||||
|
retries.
|
||||||
|
- `docs/internal/llm.md` must describe adapter contract replacement, actual
|
||||||
|
repair metadata, error adaptation, source compatibility, and scheduling.
|
||||||
|
- `docs/internal/pipeline.md` must describe how effective repair configuration
|
||||||
|
is resolved and how inner repair differs from outer stage attempts.
|
||||||
|
- `docs/policy/architecture.md` should receive only the durable ownership rule:
|
||||||
|
PromptKit owns bounded deterministic structural repair within one completion,
|
||||||
|
while Notarius owns stage attempts and semantic validation policy. Detailed
|
||||||
|
fields and retry formulas belong in their canonical configuration and
|
||||||
|
operations documents.
|
||||||
|
|
||||||
|
Update maintained configuration examples only if the public Notarius
|
||||||
|
configuration contract changes. A short inheritance illustration may remain in
|
||||||
|
the configuration reference; do not create a complete example solely to copy
|
||||||
|
PromptKit's upstream profile catalog. All upstream links must point to the
|
||||||
|
v0.8.0 tag. Historical release or archived roadmap references should remain
|
||||||
|
historical.
|
||||||
|
|
||||||
|
No ADR is required solely to pin a newer dependency. The durable separation
|
||||||
|
between PromptKit structural repair and Notarius semantic stage retries should
|
||||||
|
be stated in architecture documentation now; the more extensive future
|
||||||
|
validation state machine still warrants the separate ADR already identified in
|
||||||
|
`future.md` when that work is promoted.
|
||||||
|
|
||||||
|
## Validation And Acceptance Criteria
|
||||||
|
|
||||||
|
The implementation is complete when:
|
||||||
|
|
||||||
|
- the repository builds and tests against PromptKit v0.8.0 with no replacement
|
||||||
|
directive, workspace dependency, or vendored source;
|
||||||
|
- every maintained prompt and profile prepares or inspects successfully under
|
||||||
|
the v0.8.0 source, path, endpoint, output-contract, and JSON-value rules;
|
||||||
|
- an invalid first JSON Schema candidate followed by a valid correction returns
|
||||||
|
the valid raw output, cumulative usage, and actual repair count through the
|
||||||
|
Notarius adapter;
|
||||||
|
- repair exhaustion returns the final raw candidate and debug material with an
|
||||||
|
error matching `ErrInvalidStructuredOutput`;
|
||||||
|
- a corrective generation failure retains safe generation classification and
|
||||||
|
provider status without leaking untrusted provider detail;
|
||||||
|
- explicit empty content follows structural validation rather than being
|
||||||
|
misclassified by Notarius;
|
||||||
|
- repair configuration is presence-aware, range checked, rejected on
|
||||||
|
deterministic bindings, resolved with documented precedence, and included in
|
||||||
|
effective pipeline identity;
|
||||||
|
- inherited profiles resolve consistently during preflight and execution, and
|
||||||
|
changes to any relevant operator, fallback, or built-in parent invalidate
|
||||||
|
checkpoint reuse;
|
||||||
|
- the Rakestrawhome built-in profile reaches generic preflight, scheduling, and
|
||||||
|
provenance paths without application-specific registration;
|
||||||
|
- optional missing credentials and explicitly required credentials behave as
|
||||||
|
documented without contacting real providers in tests;
|
||||||
|
- cancellation, timeout, backend capacity, prepared-execution snapshot,
|
||||||
|
session ID, raw-output, debug-redaction, and existing profile provenance
|
||||||
|
behavior remain intact;
|
||||||
|
- maintained examples validate successfully; and
|
||||||
|
- canonical documentation contains no active v0.5.0 pin or claim that PromptKit
|
||||||
|
is always single-pass.
|
||||||
|
|
||||||
|
Tests should follow `docs/policy/testing.md`: exercise observable Notarius
|
||||||
|
contracts with offline fake clients or `httptest` boundaries, and do not copy
|
||||||
|
PromptKit's entire internal repair test suite or assert its exact correction
|
||||||
|
message prose. The dependency's internal wording is not a Notarius contract.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- Implementing Notarius's future feedback-aware semantic stage-retry loop.
|
||||||
|
- Adding the D&D combat-scene semantic validator.
|
||||||
|
- Redesigning warning policy or treating successful structural repair as a
|
||||||
|
warning.
|
||||||
|
- Adding provider transport retries or deciding which HTTP statuses should
|
||||||
|
consume a stage retry.
|
||||||
|
- Exposing PromptKit request, response, profile, validation, capacity, or error
|
||||||
|
types outside the LLM adapter.
|
||||||
|
- Changing durable artifact schemas, D&D prompt semantics, the D&D default
|
||||||
|
model, or the fixed pipeline shape.
|
||||||
|
- Reimplementing PromptKit profile inheritance, schema validation, response
|
||||||
|
bounds, repair conversations, backend admission, or provider parsing inside
|
||||||
|
Notarius.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
### 1. Default Structured-Output Repair Budget
|
||||||
|
|
||||||
|
**Decision: default to one repair attempt.** Set every maintained
|
||||||
|
eligible production prompt to `repair_attempts: 1`. One corrective call is a
|
||||||
|
strong fit for Notarius because every current production LLM response has a
|
||||||
|
strict JSON Schema contract, smaller cost-effective models are a deliberate
|
||||||
|
deployment target, and a precise structural diagnostic often makes one retry
|
||||||
|
materially more successful. The budget is paid only after a structurally
|
||||||
|
invalid candidate and remains tightly bounded.
|
||||||
|
|
||||||
|
**Alternative considered: retain zero by default.** This preserves single-pass
|
||||||
|
cost and latency and requires operators to opt in. It is preferable for an
|
||||||
|
environment where every additional request is expensive or where upstream
|
||||||
|
provider-native schema enforcement already produces negligible invalid output.
|
||||||
|
It is less suitable as the Notarius default because one malformed response can
|
||||||
|
otherwise discard substantial completed pipeline work.
|
||||||
|
|
||||||
|
**Alternative considered: default to two.** This may improve recovery for
|
||||||
|
weak models, but it doubles the worst-case corrective cost relative to the
|
||||||
|
selected default and compounds with outer stage retries. It should be an
|
||||||
|
operator choice supported by configuration, not the initial default, unless
|
||||||
|
observational evidence shows that the second correction has a worthwhile
|
||||||
|
marginal success rate.
|
||||||
|
|
||||||
|
### 2. Repair Override Scope
|
||||||
|
|
||||||
|
**Decision: support both pipeline and LLM-backed binding overrides.** Use
|
||||||
|
the presence-aware `structured_output_repair_attempts` field and precedence
|
||||||
|
defined above. A pipeline value provides the convenient one-line control the
|
||||||
|
operator requested, while a binding value permits an expensive normalizer or
|
||||||
|
future LLM-backed validator to use a deliberately different budget. This
|
||||||
|
mirrors Notarius's established pipeline/binding profile inheritance and scales
|
||||||
|
without editing embedded prompts.
|
||||||
|
|
||||||
|
**Alternative considered: support only a pipeline override.** This is smaller to
|
||||||
|
implement and document and still permits global enablement or disablement for
|
||||||
|
one pipeline. Its drawback is that one exceptional prompt cannot opt out or
|
||||||
|
request a larger budget without changing an embedded asset for every pipeline.
|
||||||
|
|
||||||
|
**Alternative considered: expose one global value under the top-level
|
||||||
|
`promptkit` configuration.** This makes client construction simple, but applies
|
||||||
|
the same budget to unrelated pipelines and leaks an execution policy into the
|
||||||
|
dependency configuration block. It is less compositional than pipeline-owned
|
||||||
|
policy and therefore not recommended.
|
||||||
|
|
||||||
|
### 3. Retention Of Provider-Supplied Generation Details
|
||||||
|
|
||||||
|
**Decision: retain status in the application-owned error contract and
|
||||||
|
retain redacted provider code, type, and message only in explicitly requested
|
||||||
|
debug traces.** Status is useful for diagnosis and future retry policy without
|
||||||
|
usually containing sensitive data. The other fields can materially explain a
|
||||||
|
400 response but may echo request or schema content, so they belong only in the
|
||||||
|
already-sensitive debug surface after Notarius redaction.
|
||||||
|
|
||||||
|
**Alternative considered: retain only HTTP status and discard all provider fields.**
|
||||||
|
This is the safest and smallest policy and still improves typed failure
|
||||||
|
handling. It sacrifices potentially decisive provider diagnostics, leaving an
|
||||||
|
operator with less information when a provider returns a terse status and the
|
||||||
|
problem cannot be reproduced easily.
|
||||||
|
|
||||||
|
**Alternative considered: include bounded provider code and type in normal
|
||||||
|
errors while keeping message debug-only.** Codes and types are often stable and
|
||||||
|
less sensitive than messages, but PromptKit explicitly classifies every
|
||||||
|
provider field as untrusted. Promoting them to ordinary output creates a
|
||||||
|
disclosure and compatibility burden that is not currently justified.
|
||||||
282
docs/roadmap/source-releases.md
Normal file
282
docs/roadmap/source-releases.md
Normal file
@@ -0,0 +1,282 @@
|
|||||||
|
# Source-Only Releases
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Implemented. Creating the first release under this procedure remains a
|
||||||
|
separate maintainer operation.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Define a repeatable, guarded release process for Notarius without taking on a
|
||||||
|
binary-distribution system that its current operator audience does not need.
|
||||||
|
The process should make an exact source revision, its compatibility impact,
|
||||||
|
and its validation status easy to identify while keeping installation in the
|
||||||
|
hands of technically capable operators and deployment automation.
|
||||||
|
|
||||||
|
The model is adapted from Weatherreporter's release procedure, but its target
|
||||||
|
is deliberately narrower: an immutable source tag and checked-in release note
|
||||||
|
are the release. Notarius does not publish executable archives or support
|
||||||
|
Windows as part of this work.
|
||||||
|
|
||||||
|
## Release Model
|
||||||
|
|
||||||
|
Notarius releases come from commits on `main` and use stable semantic-version
|
||||||
|
tags in the form `vMAJOR.MINOR.PATCH`. Prerelease tags are not part of the
|
||||||
|
initial process.
|
||||||
|
|
||||||
|
Every release has one nonempty, version-matched note at
|
||||||
|
`docs/releases/<tag>.md`. The note and every affected current-state document
|
||||||
|
must be present in the tagged commit. The Git tag and checked-in note together
|
||||||
|
are the durable release record; no separately editable release page is
|
||||||
|
required.
|
||||||
|
|
||||||
|
Published tags are immutable. A maintainer must never move, reuse, or delete a
|
||||||
|
published tag. If a published candidate is defective, the correction is made
|
||||||
|
on `main` and released under a new patch version. An unpublished local tag may
|
||||||
|
be deleted when candidate inspection finds a problem before any remote push.
|
||||||
|
|
||||||
|
Before `v1.0.0`, a minor release may intentionally change a documented CLI,
|
||||||
|
configuration, durable artifact, integration, or operating contract when its
|
||||||
|
release note explains the impact and required operator action. A patch release
|
||||||
|
must not intentionally break those documented contracts within its minor
|
||||||
|
line.
|
||||||
|
|
||||||
|
The existing `v0.1.0`, `v0.2.0`, and `v0.3.0` tags remain unchanged. They
|
||||||
|
predate this procedure and do not need retrospective release notes. The first
|
||||||
|
release made under this process establishes the release-note series.
|
||||||
|
|
||||||
|
## Source-Only Distribution
|
||||||
|
|
||||||
|
Notarius does not publish release binaries, archives, installers, container
|
||||||
|
images, package-manager entries, checksum files, or signatures. A release tag
|
||||||
|
is suitable for Go-native installation and for an operator-controlled build
|
||||||
|
from an exact checkout.
|
||||||
|
|
||||||
|
The primary installation form is:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go install \
|
||||||
|
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||||
|
```
|
||||||
|
|
||||||
|
Operator documentation should also describe cloning the repository, checking
|
||||||
|
out the tag in detached-head state, and building `./cmd/notarius` with the Go
|
||||||
|
version declared by `go.mod`. Private-module authentication and `GOPRIVATE`
|
||||||
|
configuration belong to the operator environment and must be documented by
|
||||||
|
mechanism rather than with real credentials.
|
||||||
|
|
||||||
|
Consumers such as Narratio should pin the desired Notarius tag in provisioning
|
||||||
|
or deployment configuration. They must continue to decide runtime
|
||||||
|
compatibility from Notarius's published receipt and artifact schema contracts,
|
||||||
|
not merely from the executable's product version.
|
||||||
|
|
||||||
|
Packaged binaries may be reconsidered if distribution demand, installation
|
||||||
|
friction, or a broader user audience justifies their build, signing, retention,
|
||||||
|
and platform-support costs. They are not a prerequisite for a disciplined
|
||||||
|
release process.
|
||||||
|
|
||||||
|
## Platform Policy
|
||||||
|
|
||||||
|
Linux is the supported deployment platform. Release validation must run the
|
||||||
|
test suite and the release build on Linux and must confirm that the command
|
||||||
|
builds with `CGO_ENABLED=0` for Linux `amd64` and `arm64`.
|
||||||
|
|
||||||
|
macOS is a best-effort development and testing platform. Release validation
|
||||||
|
should confirm that the command cross-compiles with `CGO_ENABLED=0` for Darwin
|
||||||
|
`amd64` and `arm64`, but the project does not promise packaged artifacts or a
|
||||||
|
separate runtime test environment for those targets.
|
||||||
|
|
||||||
|
Windows is unsupported. The release process must not require Windows builds,
|
||||||
|
Windows-specific compatibility work, or Windows documentation. Platform-
|
||||||
|
specific implementation may intentionally use Unix facilities when they are
|
||||||
|
important to Notarius's filesystem safety and operational model. Any later
|
||||||
|
decision to support Windows requires its own feature scope and validation
|
||||||
|
policy.
|
||||||
|
|
||||||
|
## Version Reporting
|
||||||
|
|
||||||
|
Add a root `notarius --version` interface for deployment diagnostics. It
|
||||||
|
prints exactly one line:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius vMAJOR.MINOR.PATCH
|
||||||
|
```
|
||||||
|
|
||||||
|
when the build has a valid release version, and:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius development
|
||||||
|
```
|
||||||
|
|
||||||
|
when no release version is available.
|
||||||
|
|
||||||
|
The implementation must obtain the main-module version from Go build
|
||||||
|
information so `go install ...@vMAJOR.MINOR.PATCH` reports the selected tag. It
|
||||||
|
must also accept an optional link-time version override so controlled builds
|
||||||
|
and release CI can identify an exact tag from a checkout. The override must be
|
||||||
|
validated and must not silently turn arbitrary text into a release version.
|
||||||
|
Ordinary unversioned checkout builds remain `development`; the release process
|
||||||
|
must not modify a tracked source constant for each release.
|
||||||
|
|
||||||
|
Version reporting is an informational product interface. It does not replace
|
||||||
|
receipt, configuration, prompt, or artifact schema versioning, and it must not
|
||||||
|
be used as the sole downstream compatibility check.
|
||||||
|
|
||||||
|
## Release Notes
|
||||||
|
|
||||||
|
Each new `docs/releases/<tag>.md` document has this minimum structure:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# Notarius vMAJOR.MINOR.PATCH
|
||||||
|
|
||||||
|
This release ...
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
```
|
||||||
|
|
||||||
|
The note should concisely explain the release's purpose, compatibility with the
|
||||||
|
preceding release, operator actions, and material user-visible, operational,
|
||||||
|
integration, and maintainer-visible changes. It should link to canonical
|
||||||
|
current-state documentation for exact contracts rather than duplicating those
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
Release notes are durable historical summaries. They must not contain
|
||||||
|
credentials, private infrastructure detail, sensitive campaign material, or
|
||||||
|
claims that are not true of the tagged candidate. A release note does not
|
||||||
|
excuse stale current-state documentation; affected canonical documents are
|
||||||
|
updated in the same candidate.
|
||||||
|
|
||||||
|
## Candidate Validation
|
||||||
|
|
||||||
|
The release procedure must provide copyable POSIX-shell guards that validate
|
||||||
|
the release version, release-note filename and heading, required note sections,
|
||||||
|
repository state, and module hygiene. Validation must be run from the Notarius
|
||||||
|
repository root with Go workspace behavior disabled.
|
||||||
|
|
||||||
|
At minimum, a candidate must pass:
|
||||||
|
|
||||||
|
- no tracked `go.work` or `go.work.sum`, no vendored tree, and no `replace`
|
||||||
|
directive in `go.mod`;
|
||||||
|
- `GOWORK=off go test -count=1 ./...`;
|
||||||
|
- `GOWORK=off go test -race -count=1 ./...`;
|
||||||
|
- `GOWORK=off go vet ./...`;
|
||||||
|
- `GOWORK=off go build ./...`;
|
||||||
|
- `GOWORK=off go mod tidy -diff`;
|
||||||
|
- `gofmt` verification for every tracked Go file;
|
||||||
|
- `git diff --check` and `git diff --cached --check`;
|
||||||
|
- validation of both maintained D&D configuration examples with their selected
|
||||||
|
pipeline;
|
||||||
|
- Linux `amd64` and `arm64` static command builds;
|
||||||
|
- best-effort Darwin `amd64` and `arm64` static command builds; and
|
||||||
|
- a focused manual or automated check that every added or changed local
|
||||||
|
Markdown link resolves.
|
||||||
|
|
||||||
|
The candidate review also checks for generated binaries, test output,
|
||||||
|
credentials, temporary files, module replacements, vendored dependencies, and
|
||||||
|
other unintended source-control content. Tests remain offline and do not call
|
||||||
|
an LLM provider or require live credentials.
|
||||||
|
|
||||||
|
## Candidate Publication
|
||||||
|
|
||||||
|
The release procedure must guard the exact commit immediately before tagging.
|
||||||
|
It requires:
|
||||||
|
|
||||||
|
- the current branch is `main`;
|
||||||
|
- the worktree and index are clean;
|
||||||
|
- the candidate commit has been pushed and exactly matches `origin/main`;
|
||||||
|
- the matching release note exists in that commit;
|
||||||
|
- no local or remote tag already uses the selected version; and
|
||||||
|
- the substantive release checks have passed for that exact candidate.
|
||||||
|
|
||||||
|
The maintainer records the exact candidate commit, creates a lightweight tag
|
||||||
|
bound explicitly to that commit, verifies the local tag target, and pushes only
|
||||||
|
that tag ref. The procedure must not recommend `git push --tags`.
|
||||||
|
|
||||||
|
After publication, the maintainer verifies that the remote tag resolves to the
|
||||||
|
guarded commit and that the release note can be read from the tagged tree. A
|
||||||
|
fresh temporary checkout or `go install ...@<tag>` must build successfully, and
|
||||||
|
the resulting command must report the expected version through `--version`.
|
||||||
|
|
||||||
|
## Validation-Only Release Automation
|
||||||
|
|
||||||
|
Add a tag-triggered Woodpecker pipeline that validates source releases without
|
||||||
|
publishing artifacts. It should:
|
||||||
|
|
||||||
|
- accept only stable semantic-version tags;
|
||||||
|
- require the version-matched release note;
|
||||||
|
- run the same substantive module, test, race, vet, build, formatting, and
|
||||||
|
whitespace checks as the documented local procedure;
|
||||||
|
- validate the maintained configuration examples;
|
||||||
|
- perform the supported and best-effort cross-build checks; and
|
||||||
|
- verify a release-version build's `notarius --version` output on the CI host.
|
||||||
|
|
||||||
|
The pipeline must not upload binaries, create archives or checksums, create or
|
||||||
|
edit a Gitea release object, or require a release API token. Local guards remain
|
||||||
|
authoritative before tag publication because CI begins only after the tag is
|
||||||
|
already remote.
|
||||||
|
|
||||||
|
If tag validation fails, preserve the published tag, fix the cause on `main`,
|
||||||
|
select a new patch version, and repeat the full process. Do not weaken tag
|
||||||
|
immutability merely because the release contains source rather than binaries.
|
||||||
|
|
||||||
|
## Documentation Ownership
|
||||||
|
|
||||||
|
In the target state:
|
||||||
|
|
||||||
|
- `docs/release.md` owns the maintainer release procedure, commands, ordering,
|
||||||
|
publication checks, and failure recovery;
|
||||||
|
- `docs/releases/` owns one historical summary per release made under the new
|
||||||
|
process;
|
||||||
|
- `docs/cli.md` owns the `--version` contract;
|
||||||
|
- `README.md` owns the shortest source-installation example and links to the
|
||||||
|
release procedure where useful;
|
||||||
|
- `docs/development.md` routes release preparation, tagging, and verification
|
||||||
|
work to `docs/release.md`;
|
||||||
|
- `docs/policy/documentation.md` assigns canonical ownership to the release
|
||||||
|
procedure and release notes;
|
||||||
|
- `docs/policy/architecture.md` records Linux support, best-effort macOS
|
||||||
|
development, unsupported Windows, and source-only distribution only if those
|
||||||
|
are judged durable development invariants rather than release mechanics; and
|
||||||
|
- `docs/operations.md` describes only installation or deployment consequences
|
||||||
|
relevant to operators and links to canonical CLI and release contracts.
|
||||||
|
|
||||||
|
Current-state documentation must not describe the new release process,
|
||||||
|
`--version`, or automated validation until the corresponding behavior exists.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- A maintainer can prepare, validate, tag, publish, and verify a source release
|
||||||
|
by following `docs/release.md` without relying on undocumented knowledge.
|
||||||
|
- Every new release has an immutable semantic-version tag and matching
|
||||||
|
checked-in release note in the tagged commit.
|
||||||
|
- The guarded candidate is clean, synchronized with `origin/main`, and passes
|
||||||
|
the documented substantive checks before tagging.
|
||||||
|
- Tag-triggered CI independently validates the published source and never
|
||||||
|
publishes binary artifacts.
|
||||||
|
- `go install` of a tagged version succeeds and `notarius --version` reports
|
||||||
|
that version; ordinary unversioned builds report `development`.
|
||||||
|
- Linux is the documented supported deployment platform, macOS has a
|
||||||
|
best-effort development build check, and Windows is explicitly unsupported.
|
||||||
|
- Downstream compatibility remains based on durable Notarius contracts rather
|
||||||
|
than the product version alone.
|
||||||
|
- Existing pre-procedure tags remain untouched and require no invented release
|
||||||
|
history.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- Publishing executable archives, installers, container images, checksums,
|
||||||
|
signatures, or package-manager entries.
|
||||||
|
- Supporting or cross-compiling for Windows.
|
||||||
|
- Creating or maintaining a mutable Gitea release page.
|
||||||
|
- Supporting prerelease tag syntax in the initial procedure.
|
||||||
|
- Automating version selection, release-note authorship, commits, or tag
|
||||||
|
creation.
|
||||||
|
- Retrospectively creating release notes for `v0.1.0` through `v0.3.0`.
|
||||||
|
- Treating a product version as a substitute for receipt, configuration,
|
||||||
|
prompt, or artifact schema compatibility.
|
||||||
2
go.mod
2
go.mod
@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0
|
gitea.maximumdirect.net/eric/promptkit v0.8.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,5 +1,5 @@
|
|||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
|
gitea.maximumdirect.net/eric/promptkit v0.8.0 h1:NGd9hDLu0UMxKbvittMrqM5Ua94eFb+kOE7UIir8l08=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
gitea.maximumdirect.net/eric/promptkit v0.8.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
36
internal/buildinfo/buildinfo.go
Normal file
36
internal/buildinfo/buildinfo.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
// Package buildinfo resolves the product version embedded in a Notarius build.
|
||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"runtime/debug"
|
||||||
|
)
|
||||||
|
|
||||||
|
var stableVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
|
||||||
|
|
||||||
|
// Override is set at link time for controlled builds.
|
||||||
|
var Override string
|
||||||
|
|
||||||
|
// Version returns the release version embedded in the build, or development
|
||||||
|
// when the build does not carry a stable release tag.
|
||||||
|
func Version() (string, error) {
|
||||||
|
buildVersion := ""
|
||||||
|
if info, ok := debug.ReadBuildInfo(); ok {
|
||||||
|
buildVersion = info.Main.Version
|
||||||
|
}
|
||||||
|
return resolve(Override, buildVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolve(override, buildVersion string) (string, error) {
|
||||||
|
if override != "" {
|
||||||
|
if !stableVersion.MatchString(override) {
|
||||||
|
return "", fmt.Errorf("build version override is not a stable release tag")
|
||||||
|
}
|
||||||
|
return override, nil
|
||||||
|
}
|
||||||
|
if stableVersion.MatchString(buildVersion) {
|
||||||
|
return buildVersion, nil
|
||||||
|
}
|
||||||
|
return "development", nil
|
||||||
|
}
|
||||||
45
internal/buildinfo/buildinfo_test.go
Normal file
45
internal/buildinfo/buildinfo_test.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package buildinfo
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestResolve(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
buildVersion string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "stable main module version", buildVersion: "v1.2.3", want: "v1.2.3"},
|
||||||
|
{name: "zero version", buildVersion: "v0.0.0", want: "v0.0.0"},
|
||||||
|
{name: "override takes precedence", override: "v2.3.4", buildVersion: "v1.2.3", want: "v2.3.4"},
|
||||||
|
{name: "invalid override", override: "version", buildVersion: "v1.2.3", wantErr: true},
|
||||||
|
{name: "override with whitespace", override: " v1.2.3", wantErr: true},
|
||||||
|
{name: "leading zero major", buildVersion: "v01.2.3", want: "development"},
|
||||||
|
{name: "leading zero minor", buildVersion: "v1.02.3", want: "development"},
|
||||||
|
{name: "leading zero patch", buildVersion: "v1.2.03", want: "development"},
|
||||||
|
{name: "build version with whitespace", buildVersion: "v1.2.3 ", want: "development"},
|
||||||
|
{name: "prerelease", buildVersion: "v1.2.3-rc.1", want: "development"},
|
||||||
|
{name: "build suffix", buildVersion: "v1.2.3+build.1", want: "development"},
|
||||||
|
{name: "pseudo version", buildVersion: "v0.0.0-20260102030405-abcdef123456", want: "development"},
|
||||||
|
{name: "development build", buildVersion: "(devel)", want: "development"},
|
||||||
|
{name: "missing build information", want: "development"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := resolve(tt.override, tt.buildVersion)
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("resolve() error = nil, want error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("resolve() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||||
@@ -38,6 +40,7 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||||
|
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -50,6 +53,33 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCommandVersionOutput(t *testing.T) {
|
||||||
|
previous := buildinfo.Override
|
||||||
|
t.Cleanup(func() { buildinfo.Override = previous })
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
wantCode int
|
||||||
|
wantStdout string
|
||||||
|
wantStderr string
|
||||||
|
}{
|
||||||
|
{name: "development", wantStdout: "notarius development\n"},
|
||||||
|
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
|
||||||
|
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buildinfo.Override = tt.override
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
|
||||||
|
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||||
environment := writeCommandConfig(t, "environment", "beta")
|
environment := writeCommandConfig(t, "environment", "beta")
|
||||||
|
|||||||
@@ -189,10 +189,18 @@ func TestMaintainedCompleteExamplePublishesRegistryBackedEntityOccurrences(t *te
|
|||||||
}
|
}
|
||||||
|
|
||||||
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
|
||||||
for _, laneID := range []string{"enemy-events", "npc-registry", "npc-occurrences", "item-registry", "item-occurrences", "location-registry", "location-occurrences"} {
|
if len(evidence) == 0 {
|
||||||
if !containsString(evidence.SelectedLanes, laneID) || !evidenceHasLane(evidence, laneID) {
|
t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
|
||||||
t.Fatalf("evidence context = %#v, want direct %s evidence", evidence, laneID)
|
}
|
||||||
|
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
|
||||||
|
for _, unit := range evidence {
|
||||||
|
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||||
|
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
|
||||||
}
|
}
|
||||||
|
if _, exists := seenEvidenceUnits[unit.ID]; exists {
|
||||||
|
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
|
||||||
|
}
|
||||||
|
seenEvidenceUnits[unit.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
requests := client.requestsFor(enemyevents.PromptID)
|
requests := client.requestsFor(enemyevents.PromptID)
|
||||||
@@ -415,17 +423,6 @@ func containsString(values []string, want string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func evidenceHasLane(value evidencecontext.Document, laneID string) bool {
|
|
||||||
for _, context := range value.Contexts {
|
|
||||||
for _, reference := range context.EvidenceRefs {
|
|
||||||
if reference.LaneID == laneID {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
|
||||||
for _, binding := range bindings {
|
for _, binding := range bindings {
|
||||||
if binding.SlotName == slotName && binding.Artifact != nil {
|
if binding.SlotName == slotName && binding.Artifact != nil {
|
||||||
|
|||||||
@@ -19,12 +19,15 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||||
@@ -311,6 +314,73 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
|||||||
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||||
t.Fatalf("prepare production scene and spell modules: %v", err)
|
t.Fatalf("prepare production scene and spell modules: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
schemaFS, err := components.assets.SchemaFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production schema assets: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
||||||
|
t.Fatalf("generic reconciliation schema asset: %v", err)
|
||||||
|
}
|
||||||
|
options, err := components.assets.PromptKitOptions()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production PromptKit options: %v", err)
|
||||||
|
}
|
||||||
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
||||||
|
})))
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production prompt engine: %v", err)
|
||||||
|
}
|
||||||
|
promptFS, err := components.assets.PromptFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("production prompt assets: %v", err)
|
||||||
|
}
|
||||||
|
type manifest struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Inputs []struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
} `yaml:"inputs"`
|
||||||
|
}
|
||||||
|
preparedPrompts := 0
|
||||||
|
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(promptFS, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var prompt manifest
|
||||||
|
if err := yaml.Unmarshal(data, &prompt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
|
||||||
|
for _, input := range prompt.Inputs {
|
||||||
|
inputs[input.Name] = promptkit.Inline(`{}`)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
|
||||||
|
}
|
||||||
|
if prepared.OutputContract.RepairAttempts != 1 {
|
||||||
|
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
|
||||||
|
}
|
||||||
|
preparedPrompts++
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if preparedPrompts == 0 {
|
||||||
|
t.Fatal("prepared no production prompts")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
||||||
|
|||||||
@@ -71,10 +71,10 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "malformed profile",
|
name: "malformed profile",
|
||||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
|
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||||
profileID: "malformed-profile",
|
profileID: "malformed-profile",
|
||||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||||
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
|
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid profile source",
|
name: "invalid profile source",
|
||||||
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
|||||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||||
|
for _, profiles := range []string{
|
||||||
|
"id: child\nbase_profile: missing\n",
|
||||||
|
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||||
|
} {
|
||||||
|
t.Run("invalid inheritance", func(t *testing.T) {
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profileID := "child"
|
||||||
|
if strings.Contains(profiles, "id: first") {
|
||||||
|
profileID = "first"
|
||||||
|
}
|
||||||
|
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
|
||||||
|
t.Fatalf("profile preflight error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
@@ -31,6 +32,7 @@ import (
|
|||||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
@@ -62,6 +64,20 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
|
if len(args) > 0 && args[0] == "--version" {
|
||||||
|
if len(args) != 1 {
|
||||||
|
fmt.Fprintln(stderr, "notarius: --version does not accept arguments")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
version, err := buildinfo.Version()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "notarius %s\n", version)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
opts, err = normalizeOptions(opts)
|
opts, err = normalizeOptions(opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -48,8 +48,15 @@ func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if info.Mode().Perm() != want {
|
mode := info.Mode().Perm()
|
||||||
t.Fatalf("%s mode = %#o, want %#o", path, info.Mode().Perm(), want)
|
if mode&^want != 0 {
|
||||||
|
t.Fatalf("%s mode = %#o, must not be broader than %#o", path, mode, want)
|
||||||
|
}
|
||||||
|
if info.IsDir() && mode&0o700 != 0o700 {
|
||||||
|
t.Fatalf("%s mode = %#o, want owner access", path, mode)
|
||||||
|
}
|
||||||
|
if !info.IsDir() && mode != want {
|
||||||
|
t.Fatalf("%s mode = %#o, want %#o", path, mode, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
entries, err := os.ReadDir(filepath.Join(runPath, "nested"))
|
||||||
|
|||||||
@@ -116,6 +116,10 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
|
|||||||
|
|
||||||
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||||
out := in
|
out := in
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
@@ -196,6 +200,10 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
|
|||||||
|
|
||||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||||
out := in
|
out := in
|
||||||
|
if in.StructuredOutputRepairAttempts != nil {
|
||||||
|
value := *in.StructuredOutputRepairAttempts
|
||||||
|
out.StructuredOutputRepairAttempts = &value
|
||||||
|
}
|
||||||
if len(in.Options) > 0 {
|
if len(in.Options) > 0 {
|
||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,23 +35,27 @@ type FilePromptKitLocalBackendConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type FilePipelineProfile struct {
|
type FilePipelineProfile struct {
|
||||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||||
Input fileModuleBinding `yaml:"input"`
|
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
Input fileModuleBinding `yaml:"input"`
|
||||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||||
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
|
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
|
||||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||||
artifactsSet bool `yaml:"-"`
|
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||||
stepsSet bool `yaml:"-"`
|
artifactsSet bool `yaml:"-"`
|
||||||
llmProfileSet bool `yaml:"-"`
|
stepsSet bool `yaml:"-"`
|
||||||
|
llmProfileSet bool `yaml:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||||
|
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
type plainFilePipelineProfile FilePipelineProfile
|
type plainFilePipelineProfile FilePipelineProfile
|
||||||
var decoded plainFilePipelineProfile
|
var decoded plainFilePipelineProfile
|
||||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||||
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||||
}, "pipeline profile")
|
}, "pipeline profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -147,12 +151,13 @@ type FileDebugConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fileModuleBinding struct {
|
type fileModuleBinding struct {
|
||||||
Module string
|
Module string
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Retries int
|
StructuredOutputRepairAttempts *int
|
||||||
Options map[string]any
|
Retries int
|
||||||
References map[string]fileReferenceSource
|
Options map[string]any
|
||||||
Validators pipeline.ValidatorOverride
|
References map[string]fileReferenceSource
|
||||||
|
Validators pipeline.ValidatorOverride
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileReferenceSource struct {
|
type fileReferenceSource struct {
|
||||||
@@ -264,6 +269,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
if b.LLMProfile == "" {
|
if b.LLMProfile == "" {
|
||||||
return fmt.Errorf("llm_profile must not be empty when set")
|
return fmt.Errorf("llm_profile must not be empty when set")
|
||||||
}
|
}
|
||||||
|
case "structured_output_repair_attempts":
|
||||||
|
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.StructuredOutputRepairAttempts = attempts
|
||||||
case "retries":
|
case "retries":
|
||||||
var retries int
|
var retries int
|
||||||
if err := valueNode.Decode(&retries); err != nil {
|
if err := valueNode.Decode(&retries); err != nil {
|
||||||
@@ -304,15 +315,56 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
|
|
||||||
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||||
return pipeline.ModuleBinding{
|
return pipeline.ModuleBinding{
|
||||||
Module: strings.TrimSpace(b.Module),
|
Module: strings.TrimSpace(b.Module),
|
||||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||||
Retries: b.Retries,
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
|
||||||
Options: cloneOptions(b.Options),
|
Retries: b.Retries,
|
||||||
References: fileReferenceSourcesToPipeline(b.References),
|
Options: cloneOptions(b.Options),
|
||||||
Validators: b.Validators,
|
References: fileReferenceSourcesToPipeline(b.References),
|
||||||
|
Validators: cloneValidatorOverride(b.Validators),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
|
||||||
|
if node.Kind != yaml.MappingNode {
|
||||||
|
return fmt.Errorf("%s must be an object", context)
|
||||||
|
}
|
||||||
|
for i := 0; i < len(node.Content); i += 2 {
|
||||||
|
if node.Content[i].Value != "structured_output_repair_attempts" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
|
||||||
|
if node.Tag == "!!null" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
|
||||||
|
}
|
||||||
|
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
|
||||||
|
}
|
||||||
|
var attempts int
|
||||||
|
if err := node.Decode(&attempts); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
|
||||||
|
}
|
||||||
|
if attempts < 0 || attempts > 3 {
|
||||||
|
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||||
|
}
|
||||||
|
return &attempts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
|
||||||
|
if attempts == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value := *attempts
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func LoadFileConfig(path string) (FileConfig, error) {
|
func LoadFileConfig(path string) (FileConfig, error) {
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -518,11 +570,12 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
profile := pipeline.PipelineProfile{
|
profile := pipeline.PipelineProfile{
|
||||||
ID: pipelineID,
|
ID: pipelineID,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
Input: filePipeline.Input.toPipelineBinding(),
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
|
||||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
Input: filePipeline.Input.toPipelineBinding(),
|
||||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||||
|
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||||
}
|
}
|
||||||
if filePipeline.Chunk != nil {
|
if filePipeline.Chunk != nil {
|
||||||
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
||||||
|
|||||||
@@ -138,6 +138,168 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
%s
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
validators:
|
||||||
|
- module: validator
|
||||||
|
structured_output_repair_attempts: 3
|
||||||
|
`
|
||||||
|
|
||||||
|
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
|
||||||
|
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
|
||||||
|
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
|
||||||
|
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := cfg.Pipelines["main"]
|
||||||
|
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["lane"]
|
||||||
|
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
fileProfile := file.Pipelines["main"]
|
||||||
|
fileLane := fileProfile.Artifacts["lane"]
|
||||||
|
*fileLane.Extract.StructuredOutputRepairAttempts = 1
|
||||||
|
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
|
||||||
|
fileProfile.Artifacts["lane"] = fileLane
|
||||||
|
file.Pipelines["main"] = fileProfile
|
||||||
|
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
|
||||||
|
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
|
||||||
|
}
|
||||||
|
cloned := cloneConfig(cfg)
|
||||||
|
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
redacted := cfg.Redacted()
|
||||||
|
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
|
||||||
|
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
summary := cfg.RedactedSummaryPayload().(Config)
|
||||||
|
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var roundTripped Config
|
||||||
|
if err := json.Unmarshal(data, &roundTripped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
roundTrippedProfile := roundTripped.Pipelines["main"]
|
||||||
|
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
|
||||||
|
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
|
||||||
|
const pipelineYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input: input
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
%s
|
||||||
|
`
|
||||||
|
const bindingYAML = `version: 4
|
||||||
|
pipelines:
|
||||||
|
main:
|
||||||
|
input:
|
||||||
|
module: input
|
||||||
|
%s
|
||||||
|
artifacts:
|
||||||
|
lane:
|
||||||
|
extract: extract
|
||||||
|
`
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
|
||||||
|
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
|
||||||
|
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
||||||
const configYAML = `version: 4
|
const configYAML = `version: 4
|
||||||
pipelines:
|
pipelines:
|
||||||
|
|||||||
@@ -116,6 +116,9 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
|||||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||||
}
|
}
|
||||||
|
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -195,6 +198,9 @@ func validateBinding(
|
|||||||
binding pipeline.ModuleBinding,
|
binding pipeline.ModuleBinding,
|
||||||
referencesAllowed bool,
|
referencesAllowed bool,
|
||||||
) error {
|
) error {
|
||||||
|
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -219,6 +225,13 @@ func validateBinding(
|
|||||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
|
||||||
|
if attempts != nil && (*attempts < 0 || *attempts > 3) {
|
||||||
|
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||||
if !override.Set {
|
if !override.Set {
|
||||||
return nil
|
return nil
|
||||||
@@ -230,6 +243,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
|
|||||||
}
|
}
|
||||||
for i, validator := range override.Validators {
|
for i, validator := range override.Validators {
|
||||||
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||||
|
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if strings.TrimSpace(validator.Module) == "" {
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
return fmt.Errorf("%s module must not be empty", context)
|
return fmt.Errorf("%s module must not be empty", context)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,13 +9,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type StructuredCompletionRequest struct {
|
type StructuredCompletionRequest struct {
|
||||||
StageName string `json:"stage_name"`
|
StageName string `json:"stage_name"`
|
||||||
PromptID string `json:"prompt_id,omitempty"`
|
PromptID string `json:"prompt_id,omitempty"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||||
Vars map[string]any `json:"vars,omitempty"`
|
Vars map[string]any `json:"vars,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type StructuredCompletionResponse struct {
|
type StructuredCompletionResponse struct {
|
||||||
@@ -26,6 +27,7 @@ type StructuredCompletionResponse struct {
|
|||||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||||
TotalTokens int `json:"total_tokens,omitempty"`
|
TotalTokens int `json:"total_tokens,omitempty"`
|
||||||
|
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||||
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,19 +62,27 @@ type LLMDebugMessage struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type LLMDebugResponse struct {
|
type LLMDebugResponse struct {
|
||||||
Content string `json:"content,omitempty"`
|
Content string `json:"content,omitempty"`
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
PromptID string `json:"prompt_id,omitempty"`
|
PromptID string `json:"prompt_id,omitempty"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||||
ModelName string `json:"model_name,omitempty"`
|
ModelName string `json:"model_name,omitempty"`
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
Validation map[string]any `json:"validation,omitempty"`
|
Validation map[string]any `json:"validation,omitempty"`
|
||||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||||
|
ProviderError *LLMDebugProviderError `json:"provider_error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type LLMDebugProviderError struct {
|
||||||
|
StatusCode int `json:"status_code,omitempty"`
|
||||||
|
Code string `json:"code,omitempty"`
|
||||||
|
Type string `json:"type,omitempty"`
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMDebugUsage struct {
|
type LLMDebugUsage struct {
|
||||||
@@ -126,11 +136,12 @@ func (set LLMInputSet) Clone() LLMInputSet {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ParseRequest struct {
|
type ParseRequest struct {
|
||||||
SourceID string `json:"source_id,omitempty"`
|
SourceID string `json:"source_id,omitempty"`
|
||||||
Path string `json:"path,omitempty"`
|
Path string `json:"path,omitempty"`
|
||||||
Raw []byte `json:"-"`
|
Raw []byte `json:"-"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type InputAdapter interface {
|
type InputAdapter interface {
|
||||||
@@ -139,12 +150,13 @@ type InputAdapter interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ChunkRequest struct {
|
type ChunkRequest struct {
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
References ReferenceSet `json:"references,omitempty"`
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkPlanResult struct {
|
type ChunkPlanResult struct {
|
||||||
@@ -282,14 +294,15 @@ type Warning struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type OutputRequest struct {
|
type OutputRequest struct {
|
||||||
Manifest artifacts.RunManifest `json:"manifest"`
|
Manifest artifacts.RunManifest `json:"manifest"`
|
||||||
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
|
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
|
||||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
|
||||||
|
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type OutputFile struct {
|
type OutputFile struct {
|
||||||
|
|||||||
@@ -9,3 +9,31 @@ var ErrInvalidStructuredOutput = errors.New("invalid structured output")
|
|||||||
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
|
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
|
||||||
// generation begins.
|
// generation begins.
|
||||||
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
|
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
|
||||||
|
|
||||||
|
// ErrLLMGeneration identifies a provider generation failure.
|
||||||
|
var ErrLLMGeneration = errors.New("LLM generation failed")
|
||||||
|
|
||||||
|
type LLMGenerationError struct {
|
||||||
|
status int
|
||||||
|
diagnostic string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
|
||||||
|
if status < 0 {
|
||||||
|
status = 0
|
||||||
|
}
|
||||||
|
return &LLMGenerationError{status: status, diagnostic: diagnostic}
|
||||||
|
}
|
||||||
|
func (e *LLMGenerationError) Error() string {
|
||||||
|
if e == nil || e.diagnostic == "" {
|
||||||
|
return ErrLLMGeneration.Error()
|
||||||
|
}
|
||||||
|
return e.diagnostic
|
||||||
|
}
|
||||||
|
func (e *LLMGenerationError) Unwrap() error { return ErrLLMGeneration }
|
||||||
|
func (e *LLMGenerationError) StatusCode() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.status
|
||||||
|
}
|
||||||
|
|||||||
@@ -34,14 +34,15 @@ type NormalizeArtifact[T any] struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedExtractionRequest struct {
|
type TypedExtractionRequest struct {
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
AmbientContext map[string]any
|
AmbientContext map[string]any
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedExtractionResult[T any] struct {
|
type TypedExtractionResult[T any] struct {
|
||||||
@@ -56,14 +57,15 @@ type Extractor[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedMergeRequest[T any] struct {
|
type TypedMergeRequest[T any] struct {
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
LaneID string
|
LaneID string
|
||||||
ExtractOutputs []ExtractArtifact[T]
|
ExtractOutputs []ExtractArtifact[T]
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedMergeResult[T any] struct {
|
type TypedMergeResult[T any] struct {
|
||||||
@@ -77,14 +79,15 @@ type Merger[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedNormalizeRequest[T any] struct {
|
type TypedNormalizeRequest[T any] struct {
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
LaneID string
|
LaneID string
|
||||||
MergeOutput MergeArtifact[T]
|
MergeOutput MergeArtifact[T]
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedNormalizeResult[T any] struct {
|
type TypedNormalizeResult[T any] struct {
|
||||||
@@ -115,20 +118,21 @@ type Normalizer[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedValidationRequest[T any] struct {
|
type TypedValidationRequest[T any] struct {
|
||||||
Stage string
|
Stage string
|
||||||
LaneID string
|
LaneID string
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
SourceID string
|
SourceID string
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
Chunk *source.Chunk
|
Metadata map[string]any
|
||||||
Chunks []source.Chunk
|
Chunk *source.Chunk
|
||||||
Ref source.SourceRef
|
Chunks []source.Chunk
|
||||||
Value T
|
Ref source.SourceRef
|
||||||
|
Value T
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedValidator[T any] interface {
|
type TypedValidator[T any] interface {
|
||||||
@@ -138,15 +142,16 @@ type TypedValidator[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ChunkValidationRequest struct {
|
type ChunkValidationRequest struct {
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
SourceID string
|
SourceID string
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
Chunks []source.Chunk
|
Metadata map[string]any
|
||||||
|
Chunks []source.Chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkValidator interface {
|
type ChunkValidator interface {
|
||||||
@@ -156,21 +161,22 @@ type ChunkValidator interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SerializedValidationRequest struct {
|
type SerializedValidationRequest struct {
|
||||||
Stage string
|
Stage string
|
||||||
LaneID string
|
LaneID string
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
SourceID string
|
SourceID string
|
||||||
SourceInput LLMInputMaterial
|
SourceInput LLMInputMaterial
|
||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Metadata map[string]any
|
StructuredOutputRepairAttempts *int
|
||||||
Chunk *source.Chunk
|
Metadata map[string]any
|
||||||
Chunks []source.Chunk
|
Chunk *source.Chunk
|
||||||
Schema ArtifactSchema
|
Chunks []source.Chunk
|
||||||
MediaType string
|
Schema ArtifactSchema
|
||||||
Content []byte
|
MediaType string
|
||||||
|
Content []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
type SerializedValidator interface {
|
type SerializedValidator interface {
|
||||||
|
|||||||
@@ -2,24 +2,8 @@
|
|||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "notarius.source.evidence_context",
|
"$id": "notarius.source.evidence_context",
|
||||||
"title": "notarius_source_evidence_context_v1",
|
"title": "notarius_source_evidence_context_v1",
|
||||||
"type": "object",
|
"type": "array",
|
||||||
"additionalProperties": false,
|
"items": {"$ref": "#/$defs/unit"},
|
||||||
"required": ["source_id", "source_digest", "window_units", "selected_lanes", "contexts"],
|
|
||||||
"properties": {
|
|
||||||
"source_id": {"type": "string", "minLength": 1},
|
|
||||||
"source_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
|
||||||
"window_units": {"type": "integer", "minimum": 0},
|
|
||||||
"selected_lanes": {
|
|
||||||
"type": "array",
|
|
||||||
"minItems": 1,
|
|
||||||
"uniqueItems": true,
|
|
||||||
"items": {"type": "string", "minLength": 1}
|
|
||||||
},
|
|
||||||
"contexts": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"$ref": "#/$defs/context"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"$defs": {
|
"$defs": {
|
||||||
"source_ref": {
|
"source_ref": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -42,25 +26,6 @@
|
|||||||
"ref": {"$ref": "#/$defs/source_ref"},
|
"ref": {"$ref": "#/$defs/source_ref"},
|
||||||
"metadata": {"type": "object", "additionalProperties": true}
|
"metadata": {"type": "object", "additionalProperties": true}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"evidence_ref": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["lane_id", "source_ref"],
|
|
||||||
"properties": {
|
|
||||||
"lane_id": {"type": "string", "minLength": 1},
|
|
||||||
"source_ref": {"$ref": "#/$defs/source_ref"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["context_ref", "evidence_refs", "units"],
|
|
||||||
"properties": {
|
|
||||||
"context_ref": {"$ref": "#/$defs/source_ref"},
|
|
||||||
"evidence_refs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evidence_ref"}},
|
|
||||||
"units": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/unit"}}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,118 +3,62 @@ package evidencecontext
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
type contribution struct {
|
type expandedRange struct {
|
||||||
laneID string
|
|
||||||
ref source.SourceRef
|
|
||||||
startPos int
|
startPos int
|
||||||
endPos int
|
endPos int
|
||||||
}
|
}
|
||||||
|
|
||||||
type expandedRange struct {
|
// Build validates projected source references, expands them by source-document
|
||||||
startPos int
|
// position, and returns their ordered union as an owned source-unit excerpt.
|
||||||
endPos int
|
|
||||||
contributions []contribution
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build validates accepted direct references, expands them by source-document
|
|
||||||
// position, and returns their deterministic context union.
|
|
||||||
func Build(request BuildRequest) (Document, error) {
|
func Build(request BuildRequest) (Document, error) {
|
||||||
if request.WindowUnits < 0 {
|
if request.WindowUnits < 0 {
|
||||||
return Document{}, fmt.Errorf("window_units must not be negative")
|
return nil, fmt.Errorf("window_units must not be negative")
|
||||||
}
|
|
||||||
lanes, err := normalizeSelectedLanes(request.SelectedLanes)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
}
|
||||||
if err := source.ValidateDocument(request.Source); err != nil {
|
if err := source.ValidateDocument(request.Source); err != nil {
|
||||||
return Document{}, fmt.Errorf("validate source document: %w", err)
|
return nil, fmt.Errorf("validate source document: %w", err)
|
||||||
}
|
}
|
||||||
digest, err := source.DigestDocument(request.Source)
|
digest, err := source.DigestDocument(request.Source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, fmt.Errorf("digest source document: %w", err)
|
return nil, fmt.Errorf("digest source document: %w", err)
|
||||||
}
|
}
|
||||||
if digest != request.Source.Digest {
|
if digest != request.Source.Digest {
|
||||||
return Document{}, fmt.Errorf("source digest does not match source document digest")
|
return nil, fmt.Errorf("source digest does not match source document digest")
|
||||||
}
|
}
|
||||||
|
|
||||||
selected := make(map[string]struct{}, len(lanes))
|
|
||||||
for _, laneID := range lanes {
|
|
||||||
selected[laneID] = struct{}{}
|
|
||||||
}
|
|
||||||
index := source.NewDocumentIndex(request.Source)
|
index := source.NewDocumentIndex(request.Source)
|
||||||
seen := make(map[evidenceKey]struct{})
|
ranges := make([]expandedRange, 0, len(request.SourceRefs))
|
||||||
contributions := make([]contribution, 0)
|
for refIndex, ref := range request.SourceRefs {
|
||||||
for laneIndex, laneEvidence := range request.LaneEvidence {
|
if err := index.ValidateRef(ref); err != nil {
|
||||||
laneID := strings.TrimSpace(laneEvidence.LaneID)
|
return nil, fmt.Errorf("source reference[%d]: %w", refIndex, err)
|
||||||
if _, ok := selected[laneID]; !ok {
|
|
||||||
return Document{}, fmt.Errorf("lane evidence[%d] lane %q is not selected", laneIndex, laneID)
|
|
||||||
}
|
}
|
||||||
for refIndex, ref := range laneEvidence.SourceRefs {
|
startPos, _ := index.Position(ref.StartUnitID)
|
||||||
if err := index.ValidateRef(ref); err != nil {
|
endPos, _ := index.Position(ref.EndUnitID)
|
||||||
return Document{}, fmt.Errorf("lane %q source reference[%d]: %w", laneID, refIndex, err)
|
ranges = append(ranges, expandedRange{
|
||||||
|
startPos: expandStart(startPos, request.WindowUnits),
|
||||||
|
endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
merged := mergeRanges(ranges)
|
||||||
|
unitCount := 0
|
||||||
|
for _, value := range merged {
|
||||||
|
unitCount += value.endPos - value.startPos + 1
|
||||||
|
}
|
||||||
|
document := make(Document, 0, unitCount)
|
||||||
|
for _, value := range merged {
|
||||||
|
for position := value.startPos; position <= value.endPos; position++ {
|
||||||
|
unit, err := cloneSourceUnit(request.Source.Units[position])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clone source unit at position %d: %w", position, err)
|
||||||
}
|
}
|
||||||
key := evidenceKey{laneID: laneID, ref: ref}
|
document = append(document, unit)
|
||||||
if _, exists := seen[key]; exists {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[key] = struct{}{}
|
|
||||||
startPos, _ := index.Position(ref.StartUnitID)
|
|
||||||
endPos, _ := index.Position(ref.EndUnitID)
|
|
||||||
contributions = append(contributions, contribution{laneID: laneID, ref: ref, startPos: expandStart(startPos, request.WindowUnits), endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits)})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return document, nil
|
||||||
sort.Slice(contributions, func(i, j int) bool { return lessContribution(contributions[i], contributions[j]) })
|
|
||||||
document := Document{
|
|
||||||
SourceID: request.Source.ID,
|
|
||||||
SourceDigest: digest,
|
|
||||||
WindowUnits: request.WindowUnits,
|
|
||||||
SelectedLanes: lanes,
|
|
||||||
Contexts: make([]Context, 0),
|
|
||||||
}
|
|
||||||
for _, rangeValue := range mergeRanges(contributions) {
|
|
||||||
context, err := buildContext(request.Source, rangeValue)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
|
||||||
document.Contexts = append(document.Contexts, context)
|
|
||||||
}
|
|
||||||
canonical, err := canonicalizeOwned(document)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, fmt.Errorf("validate evidence context: %w", err)
|
|
||||||
}
|
|
||||||
return canonical, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type evidenceKey struct {
|
|
||||||
laneID string
|
|
||||||
ref source.SourceRef
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizeSelectedLanes(values []string) ([]string, error) {
|
|
||||||
if len(values) == 0 {
|
|
||||||
return nil, fmt.Errorf("selected_lanes must not be empty")
|
|
||||||
}
|
|
||||||
seen := make(map[string]struct{}, len(values))
|
|
||||||
lanes := make([]string, 0, len(values))
|
|
||||||
for index, raw := range values {
|
|
||||||
laneID := strings.TrimSpace(raw)
|
|
||||||
if laneID == "" {
|
|
||||||
return nil, fmt.Errorf("selected_lanes[%d] must not be empty", index)
|
|
||||||
}
|
|
||||||
if _, exists := seen[laneID]; exists {
|
|
||||||
return nil, fmt.Errorf("selected_lanes lane %q is duplicated", laneID)
|
|
||||||
}
|
|
||||||
seen[laneID] = struct{}{}
|
|
||||||
lanes = append(lanes, laneID)
|
|
||||||
}
|
|
||||||
sort.Strings(lanes)
|
|
||||||
return lanes, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func expandStart(position, window int) int {
|
func expandStart(position, window int) int {
|
||||||
@@ -132,65 +76,25 @@ func expandEnd(position, length, window int) int {
|
|||||||
return position + window
|
return position + window
|
||||||
}
|
}
|
||||||
|
|
||||||
func lessContribution(left, right contribution) bool {
|
func mergeRanges(values []expandedRange) []expandedRange {
|
||||||
if left.startPos != right.startPos {
|
|
||||||
return left.startPos < right.startPos
|
|
||||||
}
|
|
||||||
if left.endPos != right.endPos {
|
|
||||||
return left.endPos < right.endPos
|
|
||||||
}
|
|
||||||
return lessEvidenceRef(EvidenceRef{LaneID: left.laneID, SourceRef: left.ref}, EvidenceRef{LaneID: right.laneID, SourceRef: right.ref})
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeRanges(values []contribution) []expandedRange {
|
|
||||||
if len(values) == 0 {
|
if len(values) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
ranges := make([]expandedRange, 0, len(values))
|
sort.Slice(values, func(i, j int) bool {
|
||||||
|
if values[i].startPos != values[j].startPos {
|
||||||
|
return values[i].startPos < values[j].startPos
|
||||||
|
}
|
||||||
|
return values[i].endPos < values[j].endPos
|
||||||
|
})
|
||||||
|
merged := make([]expandedRange, 0, len(values))
|
||||||
for _, value := range values {
|
for _, value := range values {
|
||||||
if len(ranges) == 0 || value.startPos > ranges[len(ranges)-1].endPos+1 {
|
if len(merged) == 0 || value.startPos > merged[len(merged)-1].endPos+1 {
|
||||||
ranges = append(ranges, expandedRange{startPos: value.startPos, endPos: value.endPos, contributions: []contribution{value}})
|
merged = append(merged, value)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
current := &ranges[len(ranges)-1]
|
if value.endPos > merged[len(merged)-1].endPos {
|
||||||
if value.endPos > current.endPos {
|
merged[len(merged)-1].endPos = value.endPos
|
||||||
current.endPos = value.endPos
|
|
||||||
}
|
}
|
||||||
current.contributions = append(current.contributions, value)
|
|
||||||
}
|
}
|
||||||
return ranges
|
return merged
|
||||||
}
|
|
||||||
|
|
||||||
func buildContext(document *source.SourceDocument, value expandedRange) (Context, error) {
|
|
||||||
evidenceRefs := make([]EvidenceRef, 0, len(value.contributions))
|
|
||||||
for _, contribution := range value.contributions {
|
|
||||||
evidenceRefs = append(evidenceRefs, EvidenceRef{LaneID: contribution.laneID, SourceRef: contribution.ref})
|
|
||||||
}
|
|
||||||
sort.Slice(evidenceRefs, func(i, j int) bool { return lessEvidenceRef(evidenceRefs[i], evidenceRefs[j]) })
|
|
||||||
units := make([]source.SourceUnit, 0, value.endPos-value.startPos+1)
|
|
||||||
for position := value.startPos; position <= value.endPos; position++ {
|
|
||||||
unit, err := cloneSourceUnit(document.Units[position])
|
|
||||||
if err != nil {
|
|
||||||
return Context{}, fmt.Errorf("clone source unit at position %d: %w", position, err)
|
|
||||||
}
|
|
||||||
units = append(units, unit)
|
|
||||||
}
|
|
||||||
return Context{
|
|
||||||
ContextRef: source.SourceRef{SourceID: document.ID, StartUnitID: units[0].ID, EndUnitID: units[len(units)-1].ID},
|
|
||||||
EvidenceRefs: evidenceRefs,
|
|
||||||
Units: units,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func lessEvidenceRef(left, right EvidenceRef) bool {
|
|
||||||
if left.LaneID != right.LaneID {
|
|
||||||
return left.LaneID < right.LaneID
|
|
||||||
}
|
|
||||||
if left.SourceRef.SourceID != right.SourceRef.SourceID {
|
|
||||||
return left.SourceRef.SourceID < right.SourceRef.SourceID
|
|
||||||
}
|
|
||||||
if left.SourceRef.StartUnitID != right.SourceRef.StartUnitID {
|
|
||||||
return left.SourceRef.StartUnitID < right.SourceRef.StartUnitID
|
|
||||||
}
|
|
||||||
return left.SourceRef.EndUnitID < right.SourceRef.EndUnitID
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"regexp"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
@@ -18,8 +17,6 @@ import (
|
|||||||
//go:embed assets/schemas/source_evidence_context.v1.json
|
//go:embed assets/schemas/source_evidence_context.v1.json
|
||||||
var schemaAssets embed.FS
|
var schemaAssets embed.FS
|
||||||
|
|
||||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
loadSchemaOnce sync.Once
|
loadSchemaOnce sync.Once
|
||||||
loadedSchema []byte
|
loadedSchema []byte
|
||||||
@@ -78,24 +75,24 @@ func (c *Codec) Encode(value Document) ([]byte, error) {
|
|||||||
|
|
||||||
func (c *Codec) Decode(content []byte) (Document, error) {
|
func (c *Codec) Decode(content []byte) (Document, error) {
|
||||||
if _, err := c.schemaBytes(); err != nil {
|
if _, err := c.schemaBytes(); err != nil {
|
||||||
return Document{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := validateSchemaInstance(content); err != nil {
|
if err := validateSchemaInstance(content); err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
||||||
decoder.DisallowUnknownFields()
|
decoder.DisallowUnknownFields()
|
||||||
var value Document
|
var value Document
|
||||||
if err := decoder.Decode(&value); err != nil {
|
if err := decoder.Decode(&value); err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
var trailing any
|
var trailing any
|
||||||
if err := decoder.Decode(&trailing); err != io.EOF {
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: multiple JSON values")
|
return nil, fmt.Errorf("decode evidence context: multiple JSON values")
|
||||||
}
|
}
|
||||||
canonical, err := canonicalizeOwned(value)
|
canonical, err := canonicalizeOwned(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, fmt.Errorf("decode evidence context: %w", err)
|
return nil, fmt.Errorf("decode evidence context: %w", err)
|
||||||
}
|
}
|
||||||
return canonical, nil
|
return canonical, nil
|
||||||
}
|
}
|
||||||
@@ -115,17 +112,16 @@ func loadAndCompileSchema() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
var identity struct {
|
var identity struct {
|
||||||
ID string `json:"$id"`
|
ID string `json:"$id"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Required []string `json:"required"`
|
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &identity); err != nil {
|
if err := json.Unmarshal(raw, &identity); err != nil {
|
||||||
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
|
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) {
|
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "array" {
|
||||||
loadSchemaErr = fmt.Errorf("source evidence context schema identity or required fields are invalid")
|
loadSchemaErr = fmt.Errorf("source evidence context schema identity is invalid")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
|
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
|
||||||
@@ -158,164 +154,65 @@ func validateSchemaInstance(content []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func hasRequiredFields(required []string) bool {
|
|
||||||
want := map[string]bool{"source_id": true, "source_digest": true, "window_units": true, "selected_lanes": true, "contexts": true}
|
|
||||||
for _, field := range required {
|
|
||||||
delete(want, field)
|
|
||||||
}
|
|
||||||
return len(want) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalize(value Document) (Document, error) {
|
func canonicalize(value Document) (Document, error) {
|
||||||
owned, err := clone(value)
|
owned, err := clone(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Document{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return canonicalizeOwned(owned)
|
return canonicalizeOwned(owned)
|
||||||
}
|
}
|
||||||
|
|
||||||
func canonicalizeOwned(value Document) (Document, error) {
|
func canonicalizeOwned(value Document) (Document, error) {
|
||||||
if err := requireIdentity("source_id", value.SourceID); err != nil {
|
if value == nil {
|
||||||
return Document{}, err
|
return nil, fmt.Errorf("document must be a JSON array")
|
||||||
}
|
}
|
||||||
if !digestPattern.MatchString(value.SourceDigest) {
|
seenUnitIDs := make(map[int]struct{}, len(value))
|
||||||
return Document{}, fmt.Errorf("source_digest must be a sha256 digest")
|
sourceID := ""
|
||||||
}
|
for unitIndex := range value {
|
||||||
if value.WindowUnits < 0 {
|
unit := value[unitIndex]
|
||||||
return Document{}, fmt.Errorf("window_units must not be negative")
|
|
||||||
}
|
|
||||||
if err := validateSelectedLanes(value.SelectedLanes); err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
|
||||||
if value.Contexts == nil {
|
|
||||||
value.Contexts = make([]Context, 0)
|
|
||||||
}
|
|
||||||
selected := make(map[string]struct{}, len(value.SelectedLanes))
|
|
||||||
for _, laneID := range value.SelectedLanes {
|
|
||||||
selected[laneID] = struct{}{}
|
|
||||||
}
|
|
||||||
seenUnits := make(map[int]struct{})
|
|
||||||
for contextIndex := range value.Contexts {
|
|
||||||
context, err := canonicalizeContext(value.SourceID, selected, seenUnits, value.Contexts[contextIndex], contextIndex)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, err
|
|
||||||
}
|
|
||||||
value.Contexts[contextIndex] = context
|
|
||||||
}
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateSelectedLanes(lanes []string) error {
|
|
||||||
if len(lanes) == 0 {
|
|
||||||
return fmt.Errorf("selected_lanes must not be empty")
|
|
||||||
}
|
|
||||||
for index, laneID := range lanes {
|
|
||||||
if err := requireIdentity(fmt.Sprintf("selected_lanes[%d]", index), laneID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if index > 0 && lanes[index-1] >= laneID {
|
|
||||||
return fmt.Errorf("selected_lanes must be unique and in lexical order")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalizeContext(sourceID string, selected map[string]struct{}, seenUnits map[int]struct{}, value Context, contextIndex int) (Context, error) {
|
|
||||||
prefix := fmt.Sprintf("contexts[%d]", contextIndex)
|
|
||||||
if len(value.EvidenceRefs) == 0 {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs must not be empty", prefix)
|
|
||||||
}
|
|
||||||
if len(value.Units) == 0 {
|
|
||||||
return Context{}, fmt.Errorf("%s.units must not be empty", prefix)
|
|
||||||
}
|
|
||||||
if err := validateRefIdentity(sourceID, value.ContextRef, prefix+".context_ref"); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
positions := make(map[int]int, len(value.Units))
|
|
||||||
for unitIndex := range value.Units {
|
|
||||||
unit := value.Units[unitIndex]
|
|
||||||
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
|
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
|
||||||
return Context{}, fmt.Errorf("%s.units[%d] has invalid required fields", prefix, unitIndex)
|
return nil, fmt.Errorf("units[%d] has invalid required fields", unitIndex)
|
||||||
}
|
}
|
||||||
if err := validateRefIdentity(sourceID, unit.Ref, fmt.Sprintf("%s.units[%d].ref", prefix, unitIndex)); err != nil {
|
if err := validateUnitRef(unit, unitIndex); err != nil {
|
||||||
return Context{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
if sourceID == "" {
|
||||||
return Context{}, fmt.Errorf("%s.units[%d].ref must identify unit id %d", prefix, unitIndex, unit.ID)
|
sourceID = unit.Ref.SourceID
|
||||||
|
} else if unit.Ref.SourceID != sourceID {
|
||||||
|
return nil, fmt.Errorf("units[%d].ref.source_id must match units[0].ref.source_id", unitIndex)
|
||||||
}
|
}
|
||||||
if _, exists := positions[unit.ID]; exists {
|
if _, exists := seenUnitIDs[unit.ID]; exists {
|
||||||
return Context{}, fmt.Errorf("%s.units contains duplicate unit id %d", prefix, unit.ID)
|
return nil, fmt.Errorf("units contains duplicate unit id %d", unit.ID)
|
||||||
}
|
|
||||||
if _, exists := seenUnits[unit.ID]; exists {
|
|
||||||
return Context{}, fmt.Errorf("contexts contain duplicate unit id %d", unit.ID)
|
|
||||||
}
|
|
||||||
positions[unit.ID] = unitIndex
|
|
||||||
seenUnits[unit.ID] = struct{}{}
|
|
||||||
}
|
|
||||||
if value.ContextRef.StartUnitID != value.Units[0].ID || value.ContextRef.EndUnitID != value.Units[len(value.Units)-1].ID {
|
|
||||||
return Context{}, fmt.Errorf("%s.context_ref must identify the first and last units", prefix)
|
|
||||||
}
|
|
||||||
for evidenceIndex := range value.EvidenceRefs {
|
|
||||||
evidence := value.EvidenceRefs[evidenceIndex]
|
|
||||||
if _, ok := selected[evidence.LaneID]; !ok {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs[%d].lane_id is not selected", prefix, evidenceIndex)
|
|
||||||
}
|
|
||||||
if err := requireIdentity(fmt.Sprintf("%s.evidence_refs[%d].lane_id", prefix, evidenceIndex), evidence.LaneID); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
if err := validateRefIdentity(sourceID, evidence.SourceRef, fmt.Sprintf("%s.evidence_refs[%d].source_ref", prefix, evidenceIndex)); err != nil {
|
|
||||||
return Context{}, err
|
|
||||||
}
|
|
||||||
start, startOK := positions[evidence.SourceRef.StartUnitID]
|
|
||||||
end, endOK := positions[evidence.SourceRef.EndUnitID]
|
|
||||||
if !startOK || !endOK || start > end {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs[%d].source_ref is outside context units", prefix, evidenceIndex)
|
|
||||||
}
|
|
||||||
if evidenceIndex > 0 && !lessEvidenceRef(value.EvidenceRefs[evidenceIndex-1], evidence) {
|
|
||||||
return Context{}, fmt.Errorf("%s.evidence_refs must be unique and in deterministic order", prefix)
|
|
||||||
}
|
}
|
||||||
|
seenUnitIDs[unit.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRefIdentity(sourceID string, ref source.SourceRef, field string) error {
|
func validateUnitRef(unit source.SourceUnit, unitIndex int) error {
|
||||||
if ref.SourceID != sourceID {
|
prefix := fmt.Sprintf("units[%d].ref", unitIndex)
|
||||||
return fmt.Errorf("%s.source_id does not match source_id", field)
|
if strings.TrimSpace(unit.Ref.SourceID) == "" || strings.TrimSpace(unit.Ref.SourceID) != unit.Ref.SourceID {
|
||||||
|
return fmt.Errorf("%s.source_id must be a non-empty trimmed string", prefix)
|
||||||
}
|
}
|
||||||
if ref.StartUnitID <= 0 || ref.EndUnitID <= 0 {
|
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||||
return fmt.Errorf("%s endpoints must be positive", field)
|
return fmt.Errorf("%s must identify unit id %d", prefix, unit.ID)
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireIdentity(field, value string) error {
|
|
||||||
if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value {
|
|
||||||
return fmt.Errorf("%s must be a non-empty trimmed string", field)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func clone(value Document) (Document, error) {
|
func clone(value Document) (Document, error) {
|
||||||
value.SelectedLanes = append([]string(nil), value.SelectedLanes...)
|
if value == nil {
|
||||||
if value.Contexts == nil {
|
return nil, nil
|
||||||
value.Contexts = make([]Context, 0)
|
|
||||||
} else {
|
|
||||||
contexts := make([]Context, len(value.Contexts))
|
|
||||||
for contextIndex, context := range value.Contexts {
|
|
||||||
contexts[contextIndex].ContextRef = context.ContextRef
|
|
||||||
contexts[contextIndex].EvidenceRefs = append([]EvidenceRef(nil), context.EvidenceRefs...)
|
|
||||||
contexts[contextIndex].Units = make([]source.SourceUnit, len(context.Units))
|
|
||||||
for unitIndex, unit := range context.Units {
|
|
||||||
cloned, err := cloneSourceUnit(unit)
|
|
||||||
if err != nil {
|
|
||||||
return Document{}, fmt.Errorf("clone contexts[%d].units[%d]: %w", contextIndex, unitIndex, err)
|
|
||||||
}
|
|
||||||
contexts[contextIndex].Units[unitIndex] = cloned
|
|
||||||
}
|
|
||||||
}
|
|
||||||
value.Contexts = contexts
|
|
||||||
}
|
}
|
||||||
return value, nil
|
cloned := make(Document, len(value))
|
||||||
|
for unitIndex, unit := range value {
|
||||||
|
owned, err := cloneSourceUnit(unit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clone units[%d]: %w", unitIndex, err)
|
||||||
|
}
|
||||||
|
cloned[unitIndex] = owned
|
||||||
|
}
|
||||||
|
return cloned, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
|
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package evidencecontext
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -12,126 +11,57 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildExpandsAndMergesEvidenceByDocumentPosition(t *testing.T) {
|
func TestBuildSelectsExpandedSourceUnitUnion(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
window int
|
window int
|
||||||
evidence []LaneEvidence
|
refs []source.SourceRef
|
||||||
wantUnits [][]int
|
wantIDs []int
|
||||||
wantRefs [][]EvidenceRef
|
|
||||||
}{
|
}{
|
||||||
{
|
{name: "zero window", refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{3}},
|
||||||
name: "zero window",
|
{name: "multi unit citation includes complete range", refs: []source.SourceRef{ref(3, 7)}, wantIDs: []int{3, 30, 7}},
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
{name: "non monotonic IDs use document positions", window: 1, refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{10, 3, 30}},
|
||||||
wantUnits: [][]int{{3}},
|
{name: "boundary clamping", window: 1, refs: []source.SourceRef{ref(10, 10), ref(50, 50)}, wantIDs: []int{10, 3, 7, 50}},
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
|
{name: "overlapping and adjacent windows merge", window: 1, refs: []source.SourceRef{ref(3, 3), ref(30, 30), ref(30, 30)}, wantIDs: []int{10, 3, 30, 7}},
|
||||||
},
|
{name: "adjacent expanded ranges merge", window: 1, refs: []source.SourceRef{ref(10, 10), ref(7, 7)}, wantIDs: []int{10, 3, 30, 7, 50}},
|
||||||
{
|
{name: "largest window clips without overflow", window: math.MaxInt, refs: []source.SourceRef{ref(30, 30)}, wantIDs: []int{10, 3, 30, 7, 50}},
|
||||||
name: "non monotonic ids use positions and clip boundaries",
|
{name: "no references returns an initialized empty document", wantIDs: []int{}},
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "separate gaps stay separate",
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(50, 50)}}},
|
|
||||||
wantUnits: [][]int{{10}, {50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(10, 10)}}, {{LaneID: "npcs", SourceRef: ref(50, 50)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "overlapping windows merge",
|
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}, {LaneID: "npcs", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "contiguous windows merge",
|
|
||||||
window: 1,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(7, 7)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7, 50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(7, 7)}, {LaneID: "npcs", SourceRef: ref(10, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "duplicate contributions retain unique lane attribution",
|
|
||||||
evidence: []LaneEvidence{
|
|
||||||
{LaneID: "spells", SourceRefs: []source.SourceRef{ref(30, 30), ref(30, 30)}},
|
|
||||||
{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}},
|
|
||||||
},
|
|
||||||
wantUnits: [][]int{{30}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}, {LaneID: "spells", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty contributions retain explicit empty contexts",
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs"}},
|
|
||||||
wantUnits: [][]int{},
|
|
||||||
wantRefs: [][]EvidenceRef{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "largest window clips without overflow",
|
|
||||||
window: math.MaxInt,
|
|
||||||
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}},
|
|
||||||
wantUnits: [][]int{{10, 3, 30, 7, 50}},
|
|
||||||
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}}},
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
document := testDocument(t)
|
got, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: test.window, SourceRefs: test.refs})
|
||||||
got, err := Build(BuildRequest{Source: document, WindowUnits: test.window, SelectedLanes: []string{"spells", "npcs"}, LaneEvidence: test.evidence})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Build() error = %v", err)
|
t.Fatalf("Build() error = %v", err)
|
||||||
}
|
}
|
||||||
if want := []string{"npcs", "spells"}; !reflect.DeepEqual(got.SelectedLanes, want) {
|
if got == nil {
|
||||||
t.Fatalf("SelectedLanes = %#v, want %#v", got.SelectedLanes, want)
|
t.Fatal("Build() returned a nil document")
|
||||||
}
|
}
|
||||||
if got.WindowUnits != test.window || got.SourceID != document.ID || got.SourceDigest != document.Digest {
|
if actual := unitIDs(got); !reflect.DeepEqual(actual, test.wantIDs) {
|
||||||
t.Fatalf("Build() identity = %#v, want source and window identity", got)
|
t.Fatalf("unit IDs = %#v, want %#v", actual, test.wantIDs)
|
||||||
}
|
|
||||||
if actual := contextUnitIDs(got.Contexts); !reflect.DeepEqual(actual, test.wantUnits) {
|
|
||||||
t.Fatalf("context unit ids = %#v, want %#v", actual, test.wantUnits)
|
|
||||||
}
|
|
||||||
if actual := contextEvidenceRefs(got.Contexts); !reflect.DeepEqual(actual, test.wantRefs) {
|
|
||||||
t.Fatalf("context evidence refs = %#v, want %#v", actual, test.wantRefs)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildIsStableAndOwnsSourceAndInputs(t *testing.T) {
|
func TestBuildCopiesSelectedUnitsAndMetadata(t *testing.T) {
|
||||||
document := testDocument(t)
|
document := testDocument(t)
|
||||||
refs := []source.SourceRef{ref(30, 30), ref(3, 3)}
|
first, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
request := BuildRequest{
|
|
||||||
Source: document,
|
|
||||||
WindowUnits: 1,
|
|
||||||
SelectedLanes: []string{"spells", "npcs"},
|
|
||||||
LaneEvidence: []LaneEvidence{{LaneID: "spells", SourceRefs: refs}, {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
|
|
||||||
}
|
|
||||||
first, err := Build(request)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
secondRequest := request
|
second, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
secondRequest.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}, {LaneID: "spells", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}}
|
|
||||||
second, err := Build(secondRequest)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(first, second) {
|
if !reflect.DeepEqual(first[0], document.Units[0]) {
|
||||||
t.Fatalf("Build() order differs:\nfirst: %#v\nsecond: %#v", first, second)
|
t.Fatalf("first unit = %#v, want unchanged source unit %#v", first[0], document.Units[0])
|
||||||
}
|
}
|
||||||
first.SelectedLanes[0] = "changed"
|
first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
||||||
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Build() returned metadata aliases to source document")
|
t.Fatal("Build() returned metadata aliases to the source document")
|
||||||
}
|
}
|
||||||
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
|
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
|
||||||
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Build() retained metadata aliases to source document")
|
t.Fatal("Build() retained metadata aliases to the source document")
|
||||||
}
|
|
||||||
refs[0].StartUnitID = 999
|
|
||||||
if !containsEvidenceRef(second.Contexts[0].EvidenceRefs, ref(30, 30)) {
|
|
||||||
t.Fatal("Build() retained source-reference input aliases")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,18 +72,11 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
|
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
|
||||||
{name: "blank selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{" "} }, want: "selected_lanes"},
|
|
||||||
{name: "duplicate selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{"npcs", " npcs "} }, want: "duplicated"},
|
|
||||||
{name: "unselected contribution", mutate: func(request *BuildRequest) {
|
|
||||||
request.LaneEvidence = []LaneEvidence{{LaneID: "other", SourceRefs: []source.SourceRef{ref(3, 3)}}}
|
|
||||||
}, want: "not selected"},
|
|
||||||
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
|
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
|
||||||
{name: "invalid reference", mutate: func(request *BuildRequest) {
|
{name: "invalid reference", mutate: func(request *BuildRequest) { request.SourceRefs = []source.SourceRef{ref(99, 99)} }, want: "source reference[0]"},
|
||||||
request.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(99, 99)}}}
|
|
||||||
}, want: "source reference[0]"},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
request := BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}}
|
request := BuildRequest{Source: testDocument(t), SourceRefs: []source.SourceRef{ref(3, 3)}}
|
||||||
test.mutate(&request)
|
test.mutate(&request)
|
||||||
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
|
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
t.Fatalf("Build() error = %v, want %q", err, test.want)
|
t.Fatalf("Build() error = %v, want %q", err, test.want)
|
||||||
@@ -162,7 +85,7 @@ func TestBuildRejectsInvalidInputs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
func TestCodecRoundTripsFixtureAndOwnsValues(t *testing.T) {
|
||||||
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
|
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -179,15 +102,16 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
|||||||
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
|
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
|
||||||
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
|
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
|
||||||
}
|
}
|
||||||
value.Contexts[0].Units[0].Text = "changed"
|
value[0].Text = "changed"
|
||||||
decoded, err := codec.Decode(encoded)
|
decoded, err := codec.Decode(encoded)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if decoded.Contexts[0].Units[0].Text != "The party meets Rowan." {
|
if decoded[0].Text != "The party meets Rowan." {
|
||||||
t.Fatal("Encode() retained mutable document storage")
|
t.Fatal("Encode() retained mutable document storage")
|
||||||
}
|
}
|
||||||
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
|
|
||||||
|
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -203,82 +127,52 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
first[0].Metadata["nested"].(map[string]any)["value"] = "changed"
|
||||||
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
if second[0].Metadata["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("Decode() returned metadata aliases")
|
t.Fatal("Decode() returned metadata aliases")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) {
|
func TestCodecRejectsInvalidDurablePayloads(t *testing.T) {
|
||||||
value, err := Build(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*Document)
|
content string
|
||||||
}{
|
}{
|
||||||
{name: "unsorted lanes", mutate: func(value *Document) { value.SelectedLanes = []string{"z", "a"} }},
|
{name: "null", content: "null"},
|
||||||
{name: "context range mismatch", mutate: func(value *Document) { value.Contexts[0].ContextRef.EndUnitID = 999 }},
|
{name: "wrapper object", content: `{"units":[]}`},
|
||||||
{name: "mismatched evidence source", mutate: func(value *Document) { value.Contexts[0].EvidenceRefs[0].SourceRef.SourceID = "other" }},
|
{name: "missing required unit field", content: `[{"id":1,"kind":"segment","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
|
||||||
{name: "invalid evidence range", mutate: func(value *Document) {
|
{name: "unknown unit field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1},"unknown":true}]`},
|
||||||
value.Contexts[0].EvidenceRefs[0].SourceRef.StartUnitID = 10
|
{name: "unknown reference field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1,"unknown":true}}]`},
|
||||||
}},
|
{name: "invalid self reference", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":2}}]`},
|
||||||
{name: "duplicate context unit", mutate: func(value *Document) { value.Contexts = append(value.Contexts, value.Contexts[0]) }},
|
{name: "mixed source documents", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session-one","start_unit_id":1,"end_unit_id":1}},{"id":2,"kind":"segment","text":"two","ref":{"source_id":"session-two","start_unit_id":2,"end_unit_id":2}}]`},
|
||||||
|
{name: "duplicate units", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}},{"id":1,"kind":"segment","text":"two","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`},
|
||||||
|
{name: "multiple JSON values", content: `[] []`},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
candidate, err := clone(value)
|
if _, err := New().Decode([]byte(test.content)); err == nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
test.mutate(&candidate)
|
|
||||||
if _, err := New().Encode(candidate); err == nil {
|
|
||||||
t.Fatal("Encode() error = nil, want durable model rejection")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
content, err := New().Encode(value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
mutate func(map[string]any)
|
|
||||||
}{
|
|
||||||
{name: "missing contexts", mutate: func(value map[string]any) { delete(value, "contexts") }},
|
|
||||||
{name: "null contexts", mutate: func(value map[string]any) { value["contexts"] = nil }},
|
|
||||||
{name: "unknown fixed field", mutate: func(value map[string]any) { value["unknown"] = true }},
|
|
||||||
{name: "missing units", mutate: func(value map[string]any) { delete(contextObject(value, 0), "units") }},
|
|
||||||
{name: "null evidence refs", mutate: func(value map[string]any) { contextObject(value, 0)["evidence_refs"] = nil }},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
raw := decodeJSON(t, content)
|
|
||||||
test.mutate(raw)
|
|
||||||
mutated, err := json.Marshal(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := New().Decode(mutated); err == nil {
|
|
||||||
t.Fatal("Decode() error = nil, want strict payload rejection")
|
t.Fatal("Decode() error = nil, want strict payload rejection")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if _, err := New().Decode(append(content, []byte(" {}")...)); err == nil {
|
if _, err := New().Encode(nil); err == nil {
|
||||||
t.Fatal("Decode() error = nil, want trailing JSON rejection")
|
t.Fatal("Encode(nil) error = nil, want array rejection")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSerializeUsesFixedArtifactIdentity(t *testing.T) {
|
func TestSerializeUsesFixedArtifactIdentityAndEmptyArray(t *testing.T) {
|
||||||
artifact, err := Serialize(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}})
|
artifact, err := Serialize(BuildRequest{Source: testDocument(t)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
|
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
|
||||||
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
|
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
|
||||||
}
|
}
|
||||||
|
if string(artifact.Content) != "[]" {
|
||||||
|
t.Fatalf("Serialize() content = %s, want []", artifact.Content)
|
||||||
|
}
|
||||||
decoded, err := New().Decode(artifact.Content)
|
decoded, err := New().Decode(artifact.Content)
|
||||||
if err != nil || len(decoded.Contexts) != 0 || decoded.Contexts == nil {
|
if err != nil || decoded == nil || len(decoded) != 0 {
|
||||||
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty contexts", decoded, err)
|
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty array", decoded, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,45 +200,10 @@ func ref(start, end int) source.SourceRef {
|
|||||||
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
|
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
|
||||||
}
|
}
|
||||||
|
|
||||||
func contextUnitIDs(contexts []Context) [][]int {
|
func unitIDs(units Document) []int {
|
||||||
values := make([][]int, len(contexts))
|
values := make([]int, len(units))
|
||||||
for index, context := range contexts {
|
for index, unit := range units {
|
||||||
values[index] = make([]int, len(context.Units))
|
values[index] = unit.ID
|
||||||
for unitIndex, unit := range context.Units {
|
|
||||||
values[index][unitIndex] = unit.ID
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return values
|
return values
|
||||||
}
|
}
|
||||||
|
|
||||||
func contextEvidenceRefs(contexts []Context) [][]EvidenceRef {
|
|
||||||
values := make([][]EvidenceRef, len(contexts))
|
|
||||||
for index, context := range contexts {
|
|
||||||
values[index] = append([]EvidenceRef(nil), context.EvidenceRefs...)
|
|
||||||
}
|
|
||||||
return values
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsEvidenceRef(values []EvidenceRef, want source.SourceRef) bool {
|
|
||||||
for _, value := range values {
|
|
||||||
if value.SourceRef == want {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeJSON(t *testing.T, content []byte) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
|
||||||
decoder.UseNumber()
|
|
||||||
var value map[string]any
|
|
||||||
if err := decoder.Decode(&value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
func contextObject(value map[string]any, index int) map[string]any {
|
|
||||||
return value["contexts"].([]any)[index].(map[string]any)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -14,37 +14,12 @@ const (
|
|||||||
MediaType = "application/json"
|
MediaType = "application/json"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Document is the durable union of direct evidence and surrounding source
|
// Document is the durable selected source-unit excerpt.
|
||||||
// context selected for one accepted source document.
|
type Document []source.SourceUnit
|
||||||
type Document struct {
|
|
||||||
SourceID string `json:"source_id"`
|
|
||||||
SourceDigest string `json:"source_digest"`
|
|
||||||
WindowUnits int `json:"window_units"`
|
|
||||||
SelectedLanes []string `json:"selected_lanes"`
|
|
||||||
Contexts []Context `json:"contexts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Context struct {
|
// BuildRequest supplies accepted source material and projected source references.
|
||||||
ContextRef source.SourceRef `json:"context_ref"`
|
|
||||||
EvidenceRefs []EvidenceRef `json:"evidence_refs"`
|
|
||||||
Units []source.SourceUnit `json:"units"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type EvidenceRef struct {
|
|
||||||
LaneID string `json:"lane_id"`
|
|
||||||
SourceRef source.SourceRef `json:"source_ref"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// LaneEvidence attributes direct source references to one selected lane.
|
|
||||||
type LaneEvidence struct {
|
|
||||||
LaneID string `json:"lane_id"`
|
|
||||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildRequest supplies accepted source material and direct lane evidence.
|
|
||||||
type BuildRequest struct {
|
type BuildRequest struct {
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
WindowUnits int
|
WindowUnits int
|
||||||
SelectedLanes []string
|
SourceRefs []source.SourceRef
|
||||||
LaneEvidence []LaneEvidence
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"source_id":"session-alpha","source_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","window_units":0,"selected_lanes":["npcs"],"contexts":[{"context_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13},"evidence_refs":[{"lane_id":"npcs","source_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}],"units":[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]}]}
|
[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -115,6 +116,9 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
if err := validateOutputTarget(out); err != nil {
|
if err := validateOutputTarget(out); err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, err
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
}
|
}
|
||||||
|
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
|
||||||
|
}
|
||||||
promptID := strings.TrimSpace(req.PromptID)
|
promptID := strings.TrimSpace(req.PromptID)
|
||||||
if promptID == "" {
|
if promptID == "" {
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||||
@@ -137,6 +141,18 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
Vars: promptKitVars(req, sessionID),
|
Vars: promptKitVars(req, sessionID),
|
||||||
Execution: execution,
|
Execution: execution,
|
||||||
}
|
}
|
||||||
|
if req.StructuredOutputRepairAttempts != nil {
|
||||||
|
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
|
||||||
|
if err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("inspect PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||||
|
}
|
||||||
|
contract := inspection.OutputContract
|
||||||
|
contract.RepairAttempts = *req.StructuredOutputRepairAttempts
|
||||||
|
runReq.Validation = &contract
|
||||||
|
}
|
||||||
prepared, err := c.engine.PrepareExecution(ctx, runReq)
|
prepared, err := c.engine.PrepareExecution(ctx, runReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
@@ -169,6 +185,16 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
redactPromptKitError(err),
|
redactPromptKitError(err),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
status := 0
|
||||||
|
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
|
||||||
|
if errors.As(err, &generationErr) {
|
||||||
|
status = generationErr.StatusCode()
|
||||||
|
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
|
||||||
|
}
|
||||||
|
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
|
||||||
|
}
|
||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
|
||||||
}
|
}
|
||||||
if result == nil {
|
if result == nil {
|
||||||
@@ -187,6 +213,27 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
|
||||||
|
if generationErr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var secrets []string
|
||||||
|
if prepared != nil && strings.TrimSpace(prepared.EffectiveModelParams.APIKeyEnv) != "" {
|
||||||
|
if value, ok := os.LookupEnv(prepared.EffectiveModelParams.APIKeyEnv); ok {
|
||||||
|
secrets = append(secrets, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
redact := func(value string) string {
|
||||||
|
return bearerTokenPattern.ReplaceAllString(RedactSecrets(value, secrets), "Bearer "+secretReplacement)
|
||||||
|
}
|
||||||
|
return &contracts.LLMDebugResponse{ProviderError: &contracts.LLMDebugProviderError{
|
||||||
|
StatusCode: generationErr.StatusCode(),
|
||||||
|
Code: redact(generationErr.ProviderCode()),
|
||||||
|
Type: redact(generationErr.ProviderType()),
|
||||||
|
Message: redact(generationErr.ProviderMessage()),
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
||||||
content := result.Artifact.Body
|
content := result.Artifact.Body
|
||||||
if len(content) == 0 {
|
if len(content) == 0 {
|
||||||
@@ -210,6 +257,7 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
|
|||||||
PromptTokens: result.Usage.PromptTokens,
|
PromptTokens: result.Usage.PromptTokens,
|
||||||
CompletionTokens: result.Usage.CompletionTokens,
|
CompletionTokens: result.Usage.CompletionTokens,
|
||||||
TotalTokens: result.Usage.TotalTokens,
|
TotalTokens: result.Usage.TotalTokens,
|
||||||
|
RepairAttempts: result.Validation.RepairAttempts,
|
||||||
Debug: promptKitDebugMaterial(prepared, result),
|
Debug: promptKitDebugMaterial(prepared, result),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -456,6 +457,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("inherited parent", func(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||||
|
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||||
|
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||||
@@ -531,6 +558,130 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
|
||||||
|
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
|
||||||
|
t.Fatalf("inspection = %#v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.Header.Get("Authorization"); got != "" {
|
||||||
|
t.Fatalf("Authorization = %q, want omitted", got)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.T) {
|
||||||
|
const credential = "selected-test-credential"
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
_, _ = w.Write([]byte(`{"error":{"code":"temporary","type":"provider_error","message":"marker ` + credential + ` Bearer bearer-secret"}}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "generation-profile", SessionID: "generation-error-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||||
|
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
var generation *contracts.LLMGenerationError
|
||||||
|
if !errors.As(err, &generation) || generation.StatusCode() != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("generation error = %#v", generation)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), credential) || strings.Contains(err.Error(), "marker") {
|
||||||
|
t.Fatalf("ordinary error leaked provider detail: %v", err)
|
||||||
|
}
|
||||||
|
if response.Debug == nil || response.Debug.Response == nil || response.Debug.Response.ProviderError == nil {
|
||||||
|
t.Fatalf("debug = %#v", response.Debug)
|
||||||
|
}
|
||||||
|
debug := response.Debug.Response.ProviderError
|
||||||
|
if debug.StatusCode != http.StatusServiceUnavailable || debug.Code != "temporary" || strings.Contains(debug.Message, credential) || strings.Contains(debug.Message, "bearer-secret") || !strings.Contains(debug.Message, "marker") {
|
||||||
|
t.Fatalf("debug provider error = %#v", debug)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientRejectsInvalidRepairAttemptOverride(t *testing.T) {
|
||||||
|
for _, attempts := range []int{-1, 4} {
|
||||||
|
t.Run("invalid", func(t *testing.T) {
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
var out map[string]any
|
||||||
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", StructuredOutputRepairAttempts: &attempts}, &out)
|
||||||
|
if err == nil || fake.calls != 0 {
|
||||||
|
t.Fatalf("error=%v calls=%d", err, fake.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -772,12 +923,15 @@ func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{{Content: `{"bad":true}`}, {Content: `{"bad":true}`}}}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
|
||||||
var out map[string]any
|
var out map[string]any
|
||||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
PromptID: "adapter.test",
|
PromptID: "adapter.test",
|
||||||
SessionID: "session-123",
|
SessionID: "session-123",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
Inputs: contracts.LLMInputSet{
|
Inputs: contracts.LLMInputSet{
|
||||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
},
|
},
|
||||||
@@ -785,6 +939,9 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
|||||||
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
||||||
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
||||||
}
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want exhausted repair budget", got)
|
||||||
|
}
|
||||||
if got := string(resp.Content); got != `{"bad":true}` {
|
if got := string(resp.Content); got != `{"bad":true}` {
|
||||||
t.Fatalf("response content = %q, want raw failed output", got)
|
t.Fatalf("response content = %q, want raw failed output", got)
|
||||||
}
|
}
|
||||||
@@ -796,6 +953,45 @@ func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *testing.T) {
|
||||||
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{responses: []promptkit.GenerateResponse{
|
||||||
|
{Content: `{"bad":true}`, Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8}},
|
||||||
|
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
|
||||||
|
}}
|
||||||
|
client := newTestPromptKitClient(t, fake)
|
||||||
|
|
||||||
|
var out struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
}
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
|
SessionID: "repair-test",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
if err != nil || !out.OK {
|
||||||
|
t.Fatalf("CompleteStructured() = (%#v, %v), want repaired success", response, err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
|
||||||
|
}
|
||||||
|
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
|
||||||
|
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
|
||||||
|
}
|
||||||
|
if response.Debug == nil || response.Debug.Prompt == nil || response.Debug.Response == nil {
|
||||||
|
t.Fatalf("debug = %#v, want prompt and response details", response.Debug)
|
||||||
|
}
|
||||||
|
if response.Debug.Prompt.OutputContract["repair_attempts"] != float64(1) ||
|
||||||
|
response.Debug.Response.Validation["repair_attempts"] != float64(1) ||
|
||||||
|
response.Debug.Response.Content != `{"ok":true}` ||
|
||||||
|
response.Debug.Response.Usage.TotalTokens != 26 {
|
||||||
|
t.Fatalf("debug repair result = %#v, want configured contract and repaired response", response.Debug)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
|
|
||||||
@@ -847,8 +1043,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
|||||||
t.Fatalf("error chain exposes credential: %v", err)
|
t.Fatalf("error chain exposes credential: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errors.Unwrap(err) != nil {
|
if !errors.Is(err, contracts.ErrLLMGeneration) {
|
||||||
t.Fatalf("provider failure must not expose a wrapped diagnostic: %v", err)
|
t.Fatalf("provider failure = %v, want application generation classification", err)
|
||||||
}
|
}
|
||||||
var recoveredProviderErr *credentialBearingProviderError
|
var recoveredProviderErr *credentialBearingProviderError
|
||||||
if errors.As(err, &recoveredProviderErr) {
|
if errors.As(err, &recoveredProviderErr) {
|
||||||
@@ -857,8 +1053,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
|
|||||||
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
if errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
|
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
|
||||||
}
|
}
|
||||||
if resp.Debug != nil {
|
if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Response != nil {
|
||||||
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
|
t.Fatalf("debug material = %#v, want prepared prompt without provider details", resp.Debug)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -951,19 +1147,31 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
attempts := 1
|
||||||
cancel()
|
for _, test := range []struct {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
name string
|
||||||
|
attempts *int
|
||||||
|
}{
|
||||||
|
{name: "during preparation"},
|
||||||
|
{name: "during override inspection", attempts: &attempts},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
|
|
||||||
var out map[string]any
|
var out map[string]any
|
||||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
PromptID: "adapter.test",
|
PromptID: "adapter.test",
|
||||||
Inputs: contracts.LLMInputSet{
|
StructuredOutputRepairAttempts: test.attempts,
|
||||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
Inputs: contracts.LLMInputSet{
|
||||||
},
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
}, &out)
|
},
|
||||||
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
}, &out)
|
||||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||||
|
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1071,6 +1279,66 @@ func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
|||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
|
||||||
|
attempts := 1
|
||||||
|
fake := &fakePromptKitLLM{
|
||||||
|
responses: []promptkit.GenerateResponse{
|
||||||
|
{Content: `{"bad":true}`},
|
||||||
|
{Content: `{"ok":true}`},
|
||||||
|
{Content: `{"ok":true}`},
|
||||||
|
},
|
||||||
|
block: make(chan struct{}),
|
||||||
|
}
|
||||||
|
scheduler, err := NewScheduler(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler() error = %v", err)
|
||||||
|
}
|
||||||
|
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
|
||||||
|
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var out map[string]any
|
||||||
|
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
SessionID: "repair-session",
|
||||||
|
StructuredOutputRepairAttempts: &attempts,
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
firstDone <- callErr
|
||||||
|
}()
|
||||||
|
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||||
|
|
||||||
|
secondDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var out map[string]any
|
||||||
|
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
SessionID: "queued-session",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
secondDone <- callErr
|
||||||
|
}()
|
||||||
|
close(fake.block)
|
||||||
|
if err := <-firstDone; err != nil {
|
||||||
|
t.Fatalf("repaired completion error = %v", err)
|
||||||
|
}
|
||||||
|
if err := <-secondDone; err != nil {
|
||||||
|
t.Fatalf("queued completion error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
requests := fake.requestsSnapshot()
|
||||||
|
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
|
||||||
|
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||||
|
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
||||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||||
var out map[string]any
|
var out map[string]any
|
||||||
@@ -1225,11 +1493,13 @@ func (f *switchingPromptFS) resumePromptRead() {
|
|||||||
|
|
||||||
type fakePromptKitLLM struct {
|
type fakePromptKitLLM struct {
|
||||||
content string
|
content string
|
||||||
|
responses []promptkit.GenerateResponse
|
||||||
allowEmpty bool
|
allowEmpty bool
|
||||||
err error
|
err error
|
||||||
block chan struct{}
|
block chan struct{}
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
last promptkit.GenerateRequest
|
last promptkit.GenerateRequest
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
calls int32
|
calls int32
|
||||||
inFlight int32
|
inFlight int32
|
||||||
maxInFlight int32
|
maxInFlight int32
|
||||||
@@ -1244,8 +1514,9 @@ func (*credentialBearingProviderError) Error() string {
|
|||||||
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
f.last = req
|
f.last = req
|
||||||
|
f.requests = append(f.requests, req)
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
atomic.AddInt32(&f.calls, 1)
|
call := atomic.AddInt32(&f.calls, 1)
|
||||||
current := atomic.AddInt32(&f.inFlight, 1)
|
current := atomic.AddInt32(&f.inFlight, 1)
|
||||||
for {
|
for {
|
||||||
seen := atomic.LoadInt32(&f.maxInFlight)
|
seen := atomic.LoadInt32(&f.maxInFlight)
|
||||||
@@ -1264,6 +1535,14 @@ func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateR
|
|||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, f.err
|
||||||
}
|
}
|
||||||
|
if len(f.responses) > 0 {
|
||||||
|
index := int(call - 1)
|
||||||
|
if index >= len(f.responses) {
|
||||||
|
return nil, fmt.Errorf("unexpected provider call %d", call)
|
||||||
|
}
|
||||||
|
response := f.responses[index]
|
||||||
|
return &response, nil
|
||||||
|
}
|
||||||
content := f.content
|
content := f.content
|
||||||
if content == "" && !f.allowEmpty {
|
if content == "" && !f.allowEmpty {
|
||||||
content = `{"ok":true}`
|
content = `{"ok":true}`
|
||||||
@@ -1294,3 +1573,9 @@ func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
|
|||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
return f.last
|
return f.last
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptKitLLM) requestsSnapshot() []promptkit.GenerateRequest {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return append([]promptkit.GenerateRequest(nil), f.requests...)
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const (
|
|||||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||||
// release. Update this identity when the dependency is upgraded.
|
// release. Update this identity when the dependency is upgraded.
|
||||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
|
promptKitBuiltinProfileCatalogID = "promptkit:v0.8.0:builtin-profiles"
|
||||||
)
|
)
|
||||||
|
|
||||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ type debugEvidenceContextSummary struct {
|
|||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
SelectedLanes []string `json:"selected_lanes"`
|
SelectedLanes []string `json:"selected_lanes"`
|
||||||
WindowUnits int `json:"window_units"`
|
WindowUnits int `json:"window_units"`
|
||||||
ContextCount int `json:"context_count"`
|
|
||||||
UnitCount int `json:"unit_count"`
|
UnitCount int `json:"unit_count"`
|
||||||
SourceDigest string `json:"source_digest"`
|
SourceDigest string `json:"source_digest"`
|
||||||
}
|
}
|
||||||
@@ -49,10 +48,9 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
}
|
}
|
||||||
|
|
||||||
request := evidencecontext.BuildRequest{
|
request := evidencecontext.BuildRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
||||||
SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...),
|
SourceRefs: make([]source.SourceRef, 0),
|
||||||
LaneEvidence: make([]evidencecontext.LaneEvidence, 0, len(prepared.evidencePlan.lanes)),
|
|
||||||
}
|
}
|
||||||
for _, lane := range prepared.evidencePlan.lanes {
|
for _, lane := range prepared.evidencePlan.lanes {
|
||||||
output, ok := byLane[lane.laneID]
|
output, ok := byLane[lane.laneID]
|
||||||
@@ -73,10 +71,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID)
|
return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID)
|
||||||
}
|
}
|
||||||
request.LaneEvidence = append(request.LaneEvidence, evidencecontext.LaneEvidence{
|
request.SourceRefs = append(request.SourceRefs, references...)
|
||||||
LaneID: lane.laneID,
|
|
||||||
SourceRefs: append([]source.SourceRef(nil), references...),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
document, err := evidencecontext.Build(request)
|
document, err := evidencecontext.Build(request)
|
||||||
@@ -99,13 +94,10 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo
|
|||||||
SchemaID: artifact.Schema.ID,
|
SchemaID: artifact.Schema.ID,
|
||||||
SchemaName: artifact.Schema.Name,
|
SchemaName: artifact.Schema.Name,
|
||||||
SchemaVersion: artifact.Schema.Version,
|
SchemaVersion: artifact.Schema.Version,
|
||||||
SelectedLanes: append([]string(nil), document.SelectedLanes...),
|
SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...),
|
||||||
WindowUnits: document.WindowUnits,
|
WindowUnits: prepared.evidencePlan.policy.WindowUnits,
|
||||||
ContextCount: len(document.Contexts),
|
SourceDigest: doc.Digest,
|
||||||
SourceDigest: document.SourceDigest,
|
UnitCount: len(document),
|
||||||
}
|
|
||||||
for _, context := range document.Contexts {
|
|
||||||
summary.UnitCount += len(context.Units)
|
|
||||||
}
|
}
|
||||||
return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil
|
return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,14 +97,11 @@ func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if !reflect.DeepEqual(value.SelectedLanes, []string{"alpha", "beta", "inactive"}) || len(value.Contexts) != 1 || len(value.Contexts[0].Units) != 3 {
|
if actual := []int{value[0].ID, value[1].ID, value[2].ID}; !reflect.DeepEqual(actual, []int{1, 2, 3}) {
|
||||||
t.Fatalf("evidence context = %#v, want selected union", value)
|
t.Fatalf("evidence context = %#v, want selected union", value)
|
||||||
}
|
}
|
||||||
if got := value.Contexts[0].EvidenceRefs; len(got) != 2 || got[0].LaneID != "alpha" || got[1].LaneID != "beta" {
|
|
||||||
t.Fatalf("evidence refs = %#v, want both selected lanes", got)
|
|
||||||
}
|
|
||||||
debugJSON := string(debug.json["output/evidence-context.json"])
|
debugJSON := string(debug.json["output/evidence-context.json"])
|
||||||
if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || !strings.Contains(debugJSON, `"context_count":1`) || !strings.Contains(debugJSON, `"unit_count":3`) {
|
if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || strings.Contains(debugJSON, "context_count") || !strings.Contains(debugJSON, `"unit_count":3`) {
|
||||||
t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON)
|
t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -134,7 +131,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
|
|||||||
t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected)
|
t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected)
|
||||||
}
|
}
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if len(value.Contexts) != 1 || len(value.Contexts[0].EvidenceRefs) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "present" {
|
if len(value) != 1 || value[0].ID != 1 {
|
||||||
t.Fatalf("evidence context = %#v, want present lane only", value)
|
t.Fatalf("evidence context = %#v, want present lane only", value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -232,7 +229,7 @@ func TestRunnerEvidenceContextRebuildsFromAcceptedCheckpoint(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
value := decodeCapturedEvidence(t, encoder)
|
value := decodeCapturedEvidence(t, encoder)
|
||||||
if len(value.Contexts) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "notes" {
|
if len(value) != 1 || value[0].ID != 1 {
|
||||||
t.Fatalf("evidence context = %#v, want checkpointed normalized output", value)
|
t.Fatalf("evidence context = %#v, want checkpointed normalized output", value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
|||||||
}
|
}
|
||||||
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
||||||
}
|
}
|
||||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Metadata: request.Metadata})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata})
|
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Metadata: request.Metadata})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,12 +21,13 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type ModuleBinding struct {
|
type ModuleBinding struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Retries int `json:"retries,omitempty"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
Validators ValidatorOverride `json:"validators,omitempty"`
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
|
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactReference identifies a normalized artifact produced by an earlier
|
// ArtifactReference identifies a normalized artifact produced by an earlier
|
||||||
@@ -80,19 +81,21 @@ type ValidatorOverride struct {
|
|||||||
|
|
||||||
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||||
type moduleBindingJSON struct {
|
type moduleBindingJSON struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Retries int `json:"retries,omitempty"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
|
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||||
}
|
}
|
||||||
out := moduleBindingJSON{
|
out := moduleBindingJSON{
|
||||||
Module: binding.Module,
|
Module: binding.Module,
|
||||||
LLMProfile: binding.LLMProfile,
|
LLMProfile: binding.LLMProfile,
|
||||||
Retries: binding.Retries,
|
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
|
||||||
Options: binding.Options,
|
Retries: binding.Retries,
|
||||||
References: binding.References,
|
Options: binding.Options,
|
||||||
|
References: binding.References,
|
||||||
}
|
}
|
||||||
if binding.Validators.Set {
|
if binding.Validators.Set {
|
||||||
validators := cloneModuleBindings(binding.Validators.Validators)
|
validators := cloneModuleBindings(binding.Validators.Validators)
|
||||||
@@ -101,6 +104,42 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
|||||||
return json.Marshal(out)
|
return json.Marshal(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||||
|
if binding == nil {
|
||||||
|
return fmt.Errorf("module binding must not be nil")
|
||||||
|
}
|
||||||
|
type moduleBindingJSON struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
|
Retries int `json:"retries,omitempty"`
|
||||||
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
|
Validators json.RawMessage `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
var decoded moduleBindingJSON
|
||||||
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out := ModuleBinding{
|
||||||
|
Module: decoded.Module,
|
||||||
|
LLMProfile: decoded.LLMProfile,
|
||||||
|
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
|
||||||
|
Retries: decoded.Retries,
|
||||||
|
Options: decoded.Options,
|
||||||
|
References: decoded.References,
|
||||||
|
}
|
||||||
|
if len(decoded.Validators) > 0 && string(decoded.Validators) != "null" {
|
||||||
|
var validators []ModuleBinding
|
||||||
|
if err := json.Unmarshal(decoded.Validators, &validators); err != nil {
|
||||||
|
return fmt.Errorf("decode module binding validators: %w", err)
|
||||||
|
}
|
||||||
|
out.Validators = ValidatorOverride{Set: true, Validators: validators}
|
||||||
|
}
|
||||||
|
*binding = cloneModuleBinding(out)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
Extract ModuleBinding `json:"extract"`
|
Extract ModuleBinding `json:"extract"`
|
||||||
Merge ModuleBinding `json:"merge,omitempty"`
|
Merge ModuleBinding `json:"merge,omitempty"`
|
||||||
@@ -116,14 +155,15 @@ type PipelineStepProfile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PipelineProfile struct {
|
type PipelineProfile struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Input ModuleBinding `json:"input"`
|
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
Input ModuleBinding `json:"input"`
|
||||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||||
Steps []PipelineStepProfile `json:"steps,omitempty"`
|
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||||
Output ModuleBinding `json:"output,omitempty"`
|
Steps []PipelineStepProfile `json:"steps,omitempty"`
|
||||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
Output ModuleBinding `json:"output,omitempty"`
|
||||||
|
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResolveOptions struct {
|
type ResolveOptions struct {
|
||||||
@@ -450,6 +490,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
|
if err := applyEffectiveLLMProfiles(&resolved, profile.LLMProfile, options.LLMProfileOverride); err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
|
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
@@ -1282,12 +1325,13 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
|
|||||||
return ModuleBinding{}, err
|
return ModuleBinding{}, err
|
||||||
}
|
}
|
||||||
return ModuleBinding{
|
return ModuleBinding{
|
||||||
Module: module,
|
Module: module,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
Retries: binding.Retries,
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
|
||||||
Options: cloneOptions(binding.Options),
|
Retries: binding.Retries,
|
||||||
References: references,
|
Options: cloneOptions(binding.Options),
|
||||||
Validators: cloneValidatorOverride(binding.Validators),
|
References: references,
|
||||||
|
Validators: cloneValidatorOverride(binding.Validators),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1359,6 +1403,59 @@ func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, over
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pipelineAttempts *int) error {
|
||||||
|
apply := func(stage ModuleStage, laneID, module string, binding *ModuleBinding, executionClass contracts.ExecutionClass, kind string) error {
|
||||||
|
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||||
|
if executionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
if binding.StructuredOutputRepairAttempts != nil {
|
||||||
|
if laneID == "" {
|
||||||
|
return fmt.Errorf("pipeline %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, stage, module, kind, binding.Module)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s %q assigns structured_output_repair_attempts to deterministic %s %q", resolved.ID, laneID, stage, module, kind, binding.Module)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if binding.StructuredOutputRepairAttempts == nil {
|
||||||
|
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(pipelineAttempts)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := apply(StageInput, "", resolved.Input.Module, &resolved.Input, resolved.InputExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageChunk, "", resolved.Chunk.Module, &resolved.Chunk, resolved.ChunkExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for stepIndex := range resolved.Steps {
|
||||||
|
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||||
|
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||||
|
if err := apply(StageExtract, lane.ID, lane.Extract.Module, &lane.Extract, lane.ExtractExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageMerge, lane.ID, lane.Merge.Module, &lane.Merge, lane.MergeExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := apply(StageNormalize, lane.ID, lane.Normalize.Module, &lane.Normalize, lane.NormalizeExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := apply(StageOutput, "", resolved.Output.Module, &resolved.Output, resolved.OutputExecutionClass, "module"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for chainIndex := range resolved.ValidatorChains {
|
||||||
|
chain := &resolved.ValidatorChains[chainIndex]
|
||||||
|
for validatorIndex := range chain.Validators {
|
||||||
|
validator := &chain.Validators[validatorIndex]
|
||||||
|
if err := apply(chain.Stage, chain.LaneID, chain.ModuleKey, &validator.Binding, validator.ExecutionClass, "validator"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
||||||
if len(bindings) == 0 {
|
if len(bindings) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -408,6 +408,148 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
|
||||||
|
pipelineAttempts := 2
|
||||||
|
chunkAttempts := 1
|
||||||
|
extractAttempts := 0
|
||||||
|
validatorAttempts := 3
|
||||||
|
profile := llmProfilePipeline()
|
||||||
|
profile.StructuredOutputRepairAttempts = &pipelineAttempts
|
||||||
|
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
|
||||||
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(resolved)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(resolved) error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
|
||||||
|
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
|
||||||
|
}
|
||||||
|
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
got *int
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
|
||||||
|
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
|
||||||
|
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if test.got == nil || *test.got != test.want {
|
||||||
|
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
|
||||||
|
if extractChain == nil || len(extractChain.Validators) != 1 {
|
||||||
|
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
|
||||||
|
}
|
||||||
|
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
|
||||||
|
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
|
||||||
|
}
|
||||||
|
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
|
||||||
|
if chunkChain == nil || len(chunkChain.Validators) != 1 {
|
||||||
|
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
|
||||||
|
}
|
||||||
|
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
|
||||||
|
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
pipelineAttempts = 1
|
||||||
|
chunkAttempts = 2
|
||||||
|
extractAttempts = 3
|
||||||
|
validatorAttempts = 0
|
||||||
|
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
|
||||||
|
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
|
||||||
|
}
|
||||||
|
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*PipelineProfile)
|
||||||
|
}{
|
||||||
|
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
|
||||||
|
{name: "validator", mutate: func(profile *PipelineProfile) {
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
test.mutate(&profile)
|
||||||
|
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.StructuredOutputRepairAttempts = repairAttempts(2)
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
|
||||||
|
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
|
||||||
|
}
|
||||||
|
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||||
|
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
|
||||||
|
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
|
||||||
|
digests := make(map[string]string)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
attempts *int
|
||||||
|
}{
|
||||||
|
{name: "prompt owned", attempts: nil},
|
||||||
|
{name: "disabled", attempts: repairAttempts(0)},
|
||||||
|
{name: "configured", attempts: repairAttempts(1)},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profile := llmProfilePipeline()
|
||||||
|
profile.StructuredOutputRepairAttempts = test.attempts
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
digests[test.name] = resolved.Digest
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
|
||||||
|
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||||
catalog := newProfileCatalog(t)
|
catalog := newProfileCatalog(t)
|
||||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
@@ -1843,6 +1985,10 @@ func llmProfilePipeline() PipelineProfile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func repairAttempts(value int) *int {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func llmProfileValues(profile string) map[string]string {
|
func llmProfileValues(profile string) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
"input": profile,
|
"input": profile,
|
||||||
|
|||||||
@@ -173,11 +173,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
return failOutput(output), err
|
return failOutput(output), err
|
||||||
}
|
}
|
||||||
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
|
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
|
||||||
SourceID: input.SourceID,
|
SourceID: input.SourceID,
|
||||||
Path: input.Path,
|
Path: input.Path,
|
||||||
Raw: input.RawInput,
|
Raw: input.RawInput,
|
||||||
LLMProfile: input.pipeline.Input.LLMProfile,
|
LLMProfile: input.pipeline.Input.LLMProfile,
|
||||||
Metadata: requestMetadata,
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Input.StructuredOutputRepairAttempts),
|
||||||
|
Metadata: requestMetadata,
|
||||||
})
|
})
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
_ = checkpoints.SourceFailed(adapter.Key(), ctxErr)
|
_ = checkpoints.SourceFailed(adapter.Key(), ctxErr)
|
||||||
@@ -360,14 +361,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
return failOutput(output), err
|
return failOutput(output), err
|
||||||
}
|
}
|
||||||
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||||
Manifest: output.Manifest,
|
Manifest: output.Manifest,
|
||||||
NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
|
NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
|
||||||
Rejected: cloneRejectedOutputs(output.Rejected),
|
Rejected: cloneRejectedOutputs(output.Rejected),
|
||||||
Warnings: output.Warnings,
|
Warnings: output.Warnings,
|
||||||
LLMProfile: input.pipeline.Output.LLMProfile,
|
LLMProfile: input.pipeline.Output.LLMProfile,
|
||||||
Metadata: outputMetadata,
|
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Output.StructuredOutputRepairAttempts),
|
||||||
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
Metadata: outputMetadata,
|
||||||
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
|
||||||
|
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
|
||||||
})
|
})
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
return failOutput(output), ctxErr
|
return failOutput(output), ctxErr
|
||||||
@@ -491,9 +493,9 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
|||||||
}
|
}
|
||||||
switch item.resolved.Target {
|
switch item.resolved.Target {
|
||||||
case ValidatorTargetChunk:
|
case ValidatorTargetChunk:
|
||||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks})
|
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
|
||||||
case ValidatorTargetSerialized:
|
case ValidatorTargetSerialized:
|
||||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
||||||
default:
|
default:
|
||||||
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
|
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,6 +117,38 @@ func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
|
|||||||
return prepared
|
return prepared
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
|
||||||
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
|
lane := &prepared.Steps[0].lanes[0]
|
||||||
|
producerAttempts := 2
|
||||||
|
validatorAttempts := 0
|
||||||
|
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
|
||||||
|
if len(lane.extractValidators.validators) == 0 {
|
||||||
|
t.Fatal("extract validators are empty")
|
||||||
|
}
|
||||||
|
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
|
||||||
|
|
||||||
|
var observedProducer, observedValidator *int
|
||||||
|
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||||
|
observedProducer = request.StructuredOutputRepairAttempts
|
||||||
|
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
|
||||||
|
})
|
||||||
|
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||||
|
observedValidator = target.structuredOutputRepairAttempts
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
|
||||||
|
}
|
||||||
|
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
||||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
|
|||||||
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
|
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
|
||||||
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
|
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
|
||||||
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
|
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
|
||||||
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: requestMetadata,
|
LLMProfile: input.pipeline.Chunk.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(input.pipeline.Chunk.StructuredOutputRepairAttempts), Metadata: requestMetadata,
|
||||||
})
|
})
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
|
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
|
||||||
|
|||||||
@@ -420,7 +420,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
|
|||||||
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
|
return retryAttemptResult{}, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
|
||||||
}
|
}
|
||||||
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
|
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
|
||||||
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(extractReferences), LLMProfile: lane.Extract.LLMProfile, Metadata: requestMetadata})
|
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(extractReferences), LLMProfile: lane.Extract.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(lane.Extract.StructuredOutputRepairAttempts), Metadata: requestMetadata})
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
|
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
|
||||||
return retryAttemptResult{}, terminal.record(nil, attemptErr)
|
return retryAttemptResult{}, terminal.record(nil, attemptErr)
|
||||||
|
|||||||
@@ -41,6 +41,10 @@ type observingChunkValidator struct {
|
|||||||
request contracts.ChunkValidationRequest
|
request contracts.ChunkValidationRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type observingSerializedValidator struct {
|
||||||
|
request contracts.SerializedValidationRequest
|
||||||
|
}
|
||||||
|
|
||||||
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
||||||
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
return contracts.ExecutionClassDeterministic
|
return contracts.ExecutionClassDeterministic
|
||||||
@@ -50,6 +54,15 @@ func (v *observingChunkValidator) Validate(_ context.Context, request contracts.
|
|||||||
return contracts.ValidationResult{Approved: true}, nil
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*observingSerializedValidator) Name() string { return "observing/serialized-validator" }
|
||||||
|
func (*observingSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
func (v *observingSerializedValidator) Validate(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
v.request = request
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
||||||
|
|
||||||
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
@@ -168,6 +181,33 @@ func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerForwardsDetachedRepairAttemptsToChunkValidators(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||||
|
chunkValidator := &observingChunkValidator{}
|
||||||
|
serializedValidator := &observingSerializedValidator{}
|
||||||
|
chunkAttempts := 2
|
||||||
|
serializedAttempts := 3
|
||||||
|
chunkBinding := Binding(chunkValidator.Name())
|
||||||
|
chunkBinding.StructuredOutputRepairAttempts = &chunkAttempts
|
||||||
|
serializedBinding := Binding(serializedValidator.Name())
|
||||||
|
serializedBinding.StructuredOutputRepairAttempts = &serializedAttempts
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{
|
||||||
|
{resolved: ResolvedValidator{Binding: chunkBinding, Target: ValidatorTargetChunk}, chunk: chunkValidator},
|
||||||
|
{resolved: ResolvedValidator{Binding: serializedBinding, Target: ValidatorTargetSerialized}, serialized: serializedValidator},
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := chunkValidator.request.StructuredOutputRepairAttempts; got == nil || *got != chunkAttempts || got == chunkBinding.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("chunk validator repair attempts = %v, want detached value %d", got, chunkAttempts)
|
||||||
|
}
|
||||||
|
if got := serializedValidator.request.StructuredOutputRepairAttempts; got == nil || *got != serializedAttempts || got == serializedBinding.StructuredOutputRepairAttempts {
|
||||||
|
t.Fatalf("serialized validator repair attempts = %v, want detached value %d", got, serializedAttempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
|
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
|
||||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
|
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user