81 Commits

Author SHA1 Message Date
53d9284f0b Upgrade PromptKit and prepare version 0.7.1 2026-09-22 14:31:55 +00:00
4e77725b24 Prepare version 0.7.0 and retire completed configuration plans 2026-09-14 00:54:22 +00:00
c02eec74dd Update the validator example for version 5 configuration 2026-09-14 00:42:35 +00:00
6107adb140 Identify configuration source files in load errors 2026-09-14 00:41:41 +00:00
9c5e19c225 Preserve explicit worker counts when normalizing keys 2026-09-14 00:40:40 +00:00
04da64fe32 Validate explicit configuration branches before conversion 2026-09-14 00:39:47 +00:00
5f7923c758 Record configuration normalization at sequence boundaries 2026-09-14 00:38:50 +00:00
3010a918dc Finalize ConfigKit architecture documentation 2026-09-13 17:26:57 +00:00
19f1979bac Add composed development configuration example 2026-09-13 17:24:55 +00:00
339908c1e4 Remove legacy configuration machinery 2026-09-13 17:23:00 +00:00
467bbba3e1 Add configuration inspection commands 2026-09-13 17:16:06 +00:00
92a6a4985a Switch commands to ConfigKit configuration 2026-09-13 17:12:56 +00:00
d809085406 Prepare ConfigKit validation and listing commands 2026-09-13 16:51:46 +00:00
bb922f19d9 Prepare CLI loading for ConfigKit configuration 2026-09-13 16:50:40 +00:00
320de32a61 Resolve ConfigKit paths from declaration origins 2026-09-13 16:48:44 +00:00
69295d72c9 Publish redacted ConfigKit effective configuration 2026-09-13 16:43:32 +00:00
d770e19c17 Accept ConfigKit integers in module options 2026-09-13 16:41:14 +00:00
cf17dd7bd0 Convert ConfigKit documents to runtime configuration 2026-09-13 16:39:33 +00:00
9c247cf3a2 Validate ConfigKit pipeline documents 2026-09-13 16:35:46 +00:00
b1cd919605 Apply ConfigKit operational configuration defaults 2026-09-13 16:26:18 +00:00
b822e7f510 Add ConfigKit version 5 document boundary 2026-09-13 16:18:30 +00:00
c403a38b50 Update ConfigKit adoption plan for version 0.3.0 2026-09-13 15:49:26 +00:00
00972e8d48 Plan ConfigKit adoption and clarify implementation policy 2026-09-12 02:39:03 +00:00
946fa10764 Verify deployment-managed cache permissions 2026-09-11 15:06:29 +00:00
d2afa9dbd2 Document deployment-managed cache permissions 2026-09-11 15:04:54 +00:00
7498fda030 Apply deployment permissions to checkpoints 2026-09-11 15:03:39 +00:00
ea503acd42 Respect deployment permissions for chunk plans 2026-09-11 15:01:01 +00:00
153b1e4689 Add deployment-managed cache writer 2026-09-11 14:59:27 +00:00
7d9adc77ee Plan deployment-managed cache permissions 2026-09-11 14:24:51 +00:00
77ae132e56 Prepare the v0.6.0 release 2026-08-29 12:46:18 +00:00
deebc89255 Make command line references pipeline scoped 2026-08-29 12:34:31 +00:00
f208dbe954 Make combat scene validation more reliable 2026-08-29 11:39:02 +00:00
7e626753bf Clarify semantic reconciliation candidate numbers 2026-08-29 03:16:24 +00:00
917d150279 Improve D&D validation reliability 2026-08-29 01:24:45 +00:00
4da9360d74 Improve semantic reconciliation retries 2026-08-28 18:25:00 +00:00
5cab4e512e Improve item occurrence holder corrections 2026-08-28 14:37:22 +00:00
b178f1c684 Repair reversed D&D evidence ranges 2026-08-28 13:45:21 +00:00
a2610757cd Polish combat scene validation 2026-08-28 02:26:14 +00:00
3ad34956c5 Complete combat semantics validator verification 2026-08-28 00:24:35 +00:00
22e6caa2a0 Document combat semantics validator evaluation 2026-08-28 00:23:50 +00:00
9cb7462800 Test combat semantics correction flow 2026-08-28 00:22:56 +00:00
87385b7e14 Register optional combat semantics validator 2026-08-28 00:20:59 +00:00
0ae5ea7637 Implement combat semantics validator 2026-08-28 00:19:49 +00:00
6bc883dfb6 Add combat semantics validator prompt assets 2026-08-28 00:16:23 +00:00
fcfff3ad15 Extract shared D&D combat policy 2026-08-28 00:13:44 +00:00
0f2b23dce1 Plan combat scene validation 2026-08-27 23:26:59 +00:00
61436d7c18 Prepare the v0.5.0 release 2026-08-27 19:05:13 +00:00
079d5af337 Harden diagnostic handling and warning presentation 2026-08-27 18:41:59 +00:00
1025001f20 Complete diagnostic migration verification 2026-08-27 16:54:05 +00:00
da14924a02 Document diagnostic ownership and classification 2026-08-27 16:52:58 +00:00
2065a8288b Align debug summaries with diagnostics 2026-08-27 16:51:56 +00:00
af0119cc1d Add diagnostic counts to run receipts 2026-08-27 16:50:32 +00:00
54de2b816a Publish grouped warning and diagnostic files 2026-08-27 16:43:09 +00:00
4dbbf68051 Finalize structured diagnostic aggregation 2026-08-27 16:40:17 +00:00
480680b257 Classify remaining D&D producer diagnostics 2026-08-27 16:04:32 +00:00
6a1fd7bdb6 Classify registry normalization diagnostics 2026-08-27 15:57:29 +00:00
ccba2ce3f9 Classify source relatedness as data-quality advisories 2026-08-27 15:49:53 +00:00
1f1967c8d2 Persist diagnostics in reusable pipeline state 2026-08-27 15:40:45 +00:00
5175cb0722 Classify framework process diagnostics 2026-08-27 15:33:13 +00:00
ba569594a1 Transport structured diagnostics through the pipeline 2026-08-27 15:29:10 +00:00
acb04954eb Add diagnostic contract primitives 2026-08-27 15:19:56 +00:00
610dd3d7c3 Plan warning and diagnostic reform 2026-08-27 15:06:43 +00:00
e92bcfa74c Prepare the v0.4.0 release 2026-08-27 13:30:24 +00:00
e6b2ae88d2 Strengthen validation retries and checkpoint safety 2026-08-27 11:50:37 +00:00
b2e83bd6e7 Document feedback-aware validation retries 2026-08-27 01:35:23 +00:00
7f01c3e79e Add durable validation provenance 2026-08-27 01:27:25 +00:00
7b5f4ebd42 Apply validation retries to merge and normalize 2026-08-27 01:13:32 +00:00
4bb4582695 Integrate extraction validation retries 2026-08-27 01:06:17 +00:00
7d3434e5f6 Expose typed validation reports 2026-08-27 00:37:20 +00:00
29872b2e28 Integrate chunk validation retries 2026-08-27 00:35:37 +00:00
b487d93186 Add producer attempt state machine 2026-08-27 00:27:27 +00:00
8dd7a4324d Execute complete validator chains 2026-08-27 00:18:16 +00:00
04ba87e174 Enable correction-aware semantic reconciliation 2026-08-27 00:08:05 +00:00
a26d6ed042 Enable correction-aware downstream D&D extractors 2026-08-26 23:59:05 +00:00
759d32403f Enable correction-aware foundational D&D producers 2026-08-26 23:55:23 +00:00
1a7b20c766 Adapt corrections through PromptKit 2026-08-26 23:50:24 +00:00
85a5b52be7 Declare producer correction capability 2026-08-26 23:42:11 +00:00
9d0faabf61 Add validation policy configuration 2026-08-26 23:34:06 +00:00
1c3da3e869 Add feedback-aware correction contracts 2026-08-26 23:24:20 +00:00
9abd93502f Upgrade PromptKit to v0.9.0 2026-08-26 23:12:16 +00:00
c8a29a5fa2 Plan feedback-aware validation retries 2026-08-26 18:51:36 +00:00
391 changed files with 18624 additions and 8366 deletions

View File

@@ -1,2 +1,7 @@
Please review `docs/development.md` for initial orientation in this repository Please review `docs/development.md` for initial orientation in this repository
and follow its task-specific reading guide. and follow its task-specific reading guide.
For every planning and implementation task, follow the
[`Implementation Discipline`](docs/policy/architecture.md#implementation-discipline):
keep machinery proportionate, do not invent restrictions, and do not elevate
incidental implementation details into repository policy.

View File

@@ -16,7 +16,8 @@
"type": "string" "type": "string"
}, },
"turn_kind": { "turn_kind": {
"type": "string" "type": "string",
"enum": ["turn", "reaction", "legendary_action", "lair_action", "other"]
}, },
"source_refs": { "source_refs": {
"type": "array", "type": "array",

View File

@@ -13,7 +13,10 @@
"required": ["name", "kind", "source_refs"], "required": ["name", "kind", "source_refs"],
"properties": { "properties": {
"name": {"type": "string"}, "name": {"type": "string"},
"kind": {"type": "string"}, "kind": {
"type": "string",
"enum": ["engaged", "killed", "fled", "captured", "incapacitated"]
},
"source_refs": { "source_refs": {
"type": "array", "type": "array",
"items": { "items": {

View File

@@ -18,12 +18,20 @@ when the transcript explicitly describes it being physically destroyed or
expended as a non-payment component. Use `transferred` only when possession expended as a non-payment component. Use `transferred` only when possession
moves between two distinct named party members. moves between two distinct named party members.
Return both `from` and `to` for every occurrence, using `null` when a holder does not Return both `from` and `to` for every occurrence. Use JSON `null`, not an empty
apply. For `discovered`, set both holders to `null`. For `acquired`, set `from` string, whenever a holder does not apply. Follow this holder matrix exactly:
to `null` and provide `to`; for `lost` and `consumed`, provide `from` and set
`to` to `null`; and for `transferred`, provide both holders. Use `party` only | `kind` | required `from` | required `to` |
for collective or unresolved party possession, never for either side of a | --- | --- | --- |
transfer. Do not emit a transfer for a gift, sale, or payment outside the party. | `discovered` | `null` | `null` |
| `acquired` | `null` | `party` or the named party member gaining possession |
| `lost` | `party` or the named party member losing possession | `null` |
| `consumed` | `party` or the named party member consuming the item | `null` |
| `transferred` | one named party member | a different named party member |
Use `party` only for collective or unresolved party possession, never for
either side of a transfer. Do not emit a transfer for a gift, sale, or payment
outside the party.
Ordinary non-depleting use is not an occurrence. Do not infer acquisition from a Ordinary non-depleting use is not an occurrence. Do not infer acquisition from a
discovery, or discovery from an acquisition: emit both only when each is discovery, or discovery from an acquisition: emit both only when each is

View File

@@ -13,7 +13,10 @@
"required": ["name", "kind", "quantity", "from", "to", "source_refs"], "required": ["name", "kind", "quantity", "from", "to", "source_refs"],
"properties": { "properties": {
"name": {"type": "string"}, "name": {"type": "string"},
"kind": {"type": "string"}, "kind": {
"type": "string",
"enum": ["discovered", "acquired", "lost", "consumed", "transferred"]
},
"quantity": {"type": ["integer", "null"]}, "quantity": {"type": ["integer", "null"]},
"from": {"type": ["string", "null"]}, "from": {"type": ["string", "null"]},
"to": {"type": ["string", "null"]}, "to": {"type": ["string", "null"]},

View File

@@ -5,5 +5,5 @@ evidence, similar objects, or a shared owner as sufficient.
Keep currency denominations and materially different item types separate. Keep Keep currency denominations and materially different item types separate. Keep
uncertain aliases separate. Do not infer an item property or uniqueness. uncertain aliases separate. Do not infer an item property or uniqueness.
When selecting a canonical display name, choose one supplied candidate name Set `canonical_candidate_number` to the supplied candidate number whose label
that is the clearest established designation. is the clearest established designation.

View File

@@ -5,4 +5,5 @@ nearby evidence, nested places, or generic labels as sufficient.
Keep parent and child places separate, as well as similarly named places and Keep parent and child places separate, as well as similarly named places and
uncertain aliases. uncertain aliases.
When selecting a canonical display name, prefer the clearest established name. Set `canonical_candidate_number` to the supplied candidate number whose label
is the clearest established name.

View File

@@ -16,7 +16,8 @@
"type": "string" "type": "string"
}, },
"kind": { "kind": {
"type": "string" "type": "string",
"enum": ["mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"]
}, },
"source_refs": { "source_refs": {
"type": "array", "type": "array",

View File

@@ -3,7 +3,8 @@ contextual labels and cited transcript windows. Preserve distinct individuals
even when their names are similar or their contextual descriptions are even when their names are similar or their contextual descriptions are
identical. identical.
When selecting a canonical display name, prefer a complete, stable proper name Set `canonical_candidate_number` to the supplied candidate number whose label
is the preferred canonical display name. Prefer a complete, stable proper name
over an abbreviation. Prefer an unadorned proper name over that name plus a over an abbreviation. Prefer an unadorned proper name over that name plus a
contextual class, role, title, or relationship descriptor unless the transcript contextual class, role, title, or relationship descriptor unless the transcript
establishes the descriptor as part of the person's name. A longer display name establishes the descriptor as part of the person's name. A longer display name

View File

@@ -5,9 +5,7 @@ into multiple scenes or use facts that are not supported by it.
Return one kind, one concise title, and one concise summary. Choose exactly one Return one kind, one concise title, and one concise summary. Choose exactly one
kind: kind:
- combat: active combat materially organizes the scene, including - combat: a scene classified as combat under the shared combat policy.
initiative-like exchanges or sustained hostile action. Planning a fight or
discussing a completed fight is not combat by itself.
- narrative: current-session in-world play that is not principally active - narrative: current-session in-world play that is not principally active
combat, a prior-session recap, or sustained out-of-character session combat, a prior-session recap, or sustained out-of-character session
discussion. This includes exploration, travel, dialogue, investigation, discussion. This includes exploration, travel, dialogue, investigation,
@@ -20,15 +18,13 @@ kind:
play. play.
Narrative is the default for actual current-session gameplay that does not meet Narrative is the default for actual current-session gameplay that does not meet
another definition. When the accepted chunk is mixed: another definition. When the accepted chunk has no substantive active combat:
1. use combat when active combat is a substantive central activity, even with 1. use recap when recounting a previous session is the chunk's
brief setup, rules clarification, or immediate aftermath;
2. otherwise use recap when recounting a previous session is the chunk's
primary table purpose; primary table purpose;
3. otherwise use meta when sustained out-of-character session discussion is 2. otherwise use meta when sustained out-of-character session discussion is
primary and in-world progression is no more than incidental; and primary and in-world progression is no more than incidental; and
4. use narrative for all remaining current-session in-world play. 3. use narrative for all remaining current-session in-world play.
Brief table talk, dice resolution, rules clarification, jokes, or Brief table talk, dice resolution, rules clarification, jokes, or
administrative comments do not make a gameplay scene meta. A short recollection administrative comments do not make a gameplay scene meta. A short recollection

View File

@@ -27,6 +27,8 @@ messages:
content_file: ./sharedassets/common-dnd-transcript-chunk.md content_file: ./sharedassets/common-dnd-transcript-chunk.md
cache_control: cache_control:
type: ephemeral type: ephemeral
- role: user
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
- role: user - role: user
content_file: ./instructions.md content_file: ./instructions.md
cache_control: cache_control:

View File

@@ -6,7 +6,8 @@
"required": ["kind", "title", "summary"], "required": ["kind", "title", "summary"],
"properties": { "properties": {
"kind": { "kind": {
"type": "string" "type": "string",
"enum": ["combat", "narrative", "recap", "meta"]
}, },
"title": { "title": {
"type": "string" "type": "string"

View File

@@ -0,0 +1,8 @@
Classify only whether substantive active combat occurs in the supplied
transcript chunk under the shared combat policy. Do not judge the scene title,
summary, non-combat subtype, scene boundary, or any other aspect of a scene
description.
Return `combat` when the chunk contains substantive active combat and
`non_combat` otherwise. Give a concise, transcript-grounded explanation for
the classification.

View File

@@ -0,0 +1,23 @@
id: dnd.scene_descriptions.validate_combat
version: "v1"
default_profile: dnd-extraction
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: system
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
- role: user
content_file: ./instructions.md
- role: user
content_file: ./sharedassets/common-dnd-transcript-chunk.md
cache_control:
type: ephemeral
output:
format: json
validation_mode: json_schema
schema_path: dnd_scene_combat_semantics_llm.v1.json
repair_attempts: 1

View File

@@ -0,0 +1,18 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.dnd.scene_descriptions.combat_semantics.llm",
"type": "object",
"additionalProperties": false,
"required": ["classification", "explanation"],
"properties": {
"classification": {
"type": "string",
"enum": ["combat", "non_combat"]
},
"explanation": {
"type": "string",
"minLength": 1,
"maxLength": 512
}
}
}

View File

@@ -1,6 +1,8 @@
Transcript units are the only evidence for extracted events and factual claims. Transcript units are the only evidence for extracted events and factual claims.
Every reported factual claim must be supported by cited transcript units. Use Every reported factual claim must be supported by cited transcript units. Use
integer `start_unit_id` and `end_unit_id` values from the transcript. integer `start_unit_id` and `end_unit_id` values from the transcript.
Within each range, `start_unit_id` must identify the earlier transcript unit and
`end_unit_id` the same or a later unit according to transcript order.
When supporting evidence is non-contiguous, use multiple narrow ranges rather When supporting evidence is non-contiguous, use multiple narrow ranges rather
than a broad range that bridges unrelated conversation. than a broad range that bridges unrelated conversation.

View File

@@ -0,0 +1,13 @@
Use `combat` only when substantive active combat materially organizes the
scene. Active combat includes initiative or turn exchanges, attacks, combat
spells, damage, saves, movement, or similarly sustained hostile action.
Do not use `combat` for planning or preparing for a possible fight; threats,
hostile dialogue, or a tense confrontation; immediate aftermath, looting,
healing, or discussion of a completed fight; a recap or in-world recollection
of earlier combat; or out-of-character rules discussion without active
encounter play.
When a chunk contains substantive active combat alongside brief setup, rules
clarification, interruption, phase transition, or immediate aftermath, classify
it as `combat`.

View File

@@ -1,3 +1,3 @@
Candidate material: Candidate material, including the exact valid candidate-number range:
{{ input "candidates" }} {{ input "candidates" }}

View File

@@ -1,7 +1,9 @@
Use only the positive integer `candidate_id` values supplied in the candidate material. Use only the positive integer `candidate_number` values supplied in the candidate material. Valid candidate numbers are exactly the inclusive `first` through `last` values declared in `candidate_number_range`; use the explicit number attached to each candidate.
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group. Transcript unit `id` values and evidence `start_unit_id` and `end_unit_id` values identify source positions. They are not candidate numbers and must never be used in `candidate_numbers` or `canonical_candidate_number`.
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid. Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate numbers, and its `canonical_candidate_number` must be one of those numbers. A candidate may appear in at most one group.
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response. Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer a number that is not explicitly supplied. An empty `duplicate_groups` array is valid.
The response must conform exactly to the selected JSON schema. Return candidate numbers only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.

View File

@@ -11,9 +11,9 @@
"items": { "items": {
"type": "object", "type": "object",
"additionalProperties": false, "additionalProperties": false,
"required": ["candidate_ids", "canonical_candidate_id"], "required": ["candidate_numbers", "canonical_candidate_number"],
"properties": { "properties": {
"candidate_ids": { "candidate_numbers": {
"type": "array", "type": "array",
"minItems": 2, "minItems": 2,
"items": { "items": {
@@ -21,7 +21,7 @@
"minimum": 1 "minimum": 1
} }
}, },
"canonical_candidate_id": { "canonical_candidate_number": {
"type": "integer", "type": "integer",
"minimum": 1 "minimum": 1
} }

View File

@@ -83,9 +83,7 @@ entity identity, checkpoint contracts, or cross-request correlation. Changes
to shared protocol and policy assets must participate in the normal prompt, to shared protocol and policy assets must participate in the normal prompt,
schema, and checkpoint fingerprint mechanisms. schema, and checkpoint fingerprint mechanisms.
Acceptance of this decision does not imply that the shared mechanism or its The shared mechanism and its initial D&D registry consumers are now
consumer migrations are implemented. The implemented. Current behavior is documented in
[feature roadmap](../roadmap/semantic-reconciliation.md) owns target behavior [Module Internals](../internal/modules.md#semantic-reconciliation) and
and status, and the [D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
[implementation plan](../roadmap/implementation.md) owns delivery sequence
until the work is complete.

View File

@@ -0,0 +1,59 @@
# ADR-0014: Use feedback-aware validation retries
**Status:** Accepted
**Date:** 2026-08-26
## Context
Validation can identify a candidate defect after a producer has returned an
otherwise well-formed result. Retrying without the validator's deterministic,
bounded feedback wastes the useful diagnosis, while treating validator
execution failures as defects would ask a producer to repair conditions it
cannot control. The mechanism must preserve typed producer ownership,
checkpoint safety, and the repository's sensitive-data boundaries.
## Decision
The implementation will keep three independent budgets: the producer binding's
outer `retries` budget, PromptKit's structured-output repair budget, and each
validator's execution-retry budget. Validators will run sequentially in their
configured order and aggregate both rejections and execution failures before a
candidate disposition is selected.
A correction-capable producer will provide the exact single LLM response that
controlled its candidate using the `single_response_v1` protocol. A correction
attempt will reconstruct the ordinary request and append exactly two fresh
messages: that latest response as `assistant`, followed by one deterministic
aggregate correction request as `user`. Earlier turns will not accumulate.
Validator failures will not recurse into correction. Pipeline policy owns
terminal disposition, with field-by-field producer overrides over pipeline
defaults: structural failure and semantic rejection default to `fail_run`, and
validator execution failure defaults to `warn_continue`. Validators can report
facts and bounded corrective guidance, but never decide disposition.
Rejected and structurally invalid candidates will not advance. A candidate
allowed through after a validator execution failure will retain explicit
incomplete-validation provenance and will not be checkpointed. Exact response
and correction text remain attempt-local: they are excluded from ordinary
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
summaries.
## Alternatives considered
- Retry every producer after any validation outcome. This conflates producer
defects with validator operational failures and wastes retry budget.
- Let validators decide whether to continue. This would distribute pipeline
disposition policy across validators and undermine consistent defaults.
- Reuse the full prior conversation. Accumulated turns introduce unbounded
prompt growth and make correction behavior depend on incidental history.
- Persist raw responses to simplify diagnosis. Raw model output and correction
guidance may be sensitive and do not belong in durable pipeline records.
## Consequences
The framework gains transport-neutral correction and candidate contracts,
producer capability checks, policy resolution, aggregated validation outcomes,
and conservative checkpoint handling. Prompt construction remains inside the
LLM adapter, while modules remain responsible for accurately exposing the
single response that directly controlled a candidate.

View File

@@ -0,0 +1,67 @@
# ADR-0015: Separate process warnings from quality diagnostics
**Status:** Accepted
**Date:** 2026-08-27
## Context
Notarius currently represents process degradation, incomplete validation,
extraction-quality doubt, and routine normalization with one flat warning
record. That makes ordinary successful runs noisy, loses the framework context
needed to explain a finding, and gives `warning_count` no stable operational
meaning. It also permits output encoders to add a warning after the durable
warning file has already been written.
The application needs one bounded diagnostic model that preserves exact
occurrence counts while retaining only safe, representative samples. Fresh and
resumed logical runs must present the same groups. The model must not alter
validation decisions, retry budgets, rejected-output behavior, or process exit
policy.
## Decision
Warnings are reserved for a completed run that advanced under an allowed
process-level degradation or incomplete-work policy. Extraction-quality signals
are advisories, and routine accepted transformations are observations. A
non-degraded successful run therefore has zero actionable warnings.
Modules and validators own a diagnostic's disposition, category, reason code,
scope, and safe message. The framework adds pipeline origin, including stage,
step, lane, module, validator, and chunk context where applicable. It then
aggregates deterministically by disposition, category, reason code, and full
origin. Chunk context remains on representative samples so equivalent findings
across chunks aggregate together.
Diagnostics carry exact occurrence counts, at most three distinct samples, and
numeric omitted-sample metadata. Producers and validators are bounded to 64
local groups. Final actionable warning groups are bounded without truncation;
the non-warning collection may truncate represented groups while preserving an
exact total occurrence count and explicit truncation metadata.
The public contracts will be versioned: grouped actionable warnings use
`notarius.warnings.v2`, grouped advisories and observations use
`notarius.diagnostics.v1`, and the run receipt uses
`notarius.run-result.v2`. Successful output encoders return logical files or
an error; they do not add post-encoding warnings.
## Alternatives considered
- Keep one warning list and filter only CLI output. This would leave durable
consumers with the same semantically mixed, unbounded contract.
- Map reason codes to severity in a central framework registry. This would
split module-owned meaning between synchronized policy tables and make new
diagnostic meaning implicit.
- Preserve local omission warning records. They inflate visible group counts
and lose exact occurrence semantics.
- Keep output-encoder warnings. A one-pass encoder cannot include those
records consistently in files it has already serialized; a two-phase encoder
protocol is deferred until a demonstrated need exists.
## Consequences
The framework gains validated diagnostic primitives, local collection,
origin-aware aggregation, and versioned durable presentation. Existing warning
transport remains temporarily while producers migrate. Current architecture,
operator, integration, and internal documentation will describe the behavior
only as each implementation step lands; this accepted decision does not claim
that the migration is complete.

View File

@@ -0,0 +1,76 @@
# ADR-0016: Use feedback-aware module-requested retries
**Status:** Accepted
**Date:** 2026-08-28
## Context
An LLM-backed module can discover that a structurally valid model response is
unsafe while translating it into a typed candidate, before the ordinary
validator chain runs. Semantic registry reconciliation is the first such case:
the shared core can identify invalid duplicate-group proposals, and typed
application can reject a domain-incompatible group while preserving a safe
partial result. Repeating the original request without the rejected response or
corrective explanation gives the model no information with which to improve.
The existing feedback-aware validation mechanism already establishes the safe
correction protocol, but module-owned retry directives also carry internal
reason codes, operator messages, and fallback diagnostics. Those values are not
model instructions. Some module retry conditions, including exhausted
structured-output repair, also have no valid exact response to append.
## Decision
An LLM-backed normalizer may attach optional, bounded model-facing correction
guidance to a module-requested retry. Guidance is a separate contract field; the
framework never derives it from a reason code, operator message, diagnostic, or
error. A feedback-capable directive must include the exact model candidate that
controlled the safe fallback and must use `single_response_v1`.
The artifact-neutral producer-attempt state machine constructs the correction
from that exact latest response and the supplied guidance. The producer rebuilds
its complete ordinary request and appends the response as one assistant message
and the guidance as one user message. Earlier attempts do not accumulate, and
the attempt consumes the same configured stage retry budget as operational,
structural, validation, and feedback-free module retries.
A module retry without guidance remains valid and starts a fresh attempt. This
is the required behavior when no valid model candidate exists, including after
structured-output repair is exhausted. If feedback is supplied without a valid
supported candidate, the framework fails the module contract rather than
retrying blindly or inventing guidance.
After retry exhaustion, the normalizer's safe fallback continues through its
validator chain. Safe semantic groups may remain applied, unsafe groups remain
separate, and bounded fallback diagnostics may explain the process degradation.
Exact responses and correction text remain attempt-local and are excluded from
ordinary errors, warnings, manifests, receipts, caches, and checkpoints.
This decision extends, rather than supersedes,
[ADR-0014](0014-feedback-aware-validation-retries.md): both correction sources
use the same transport-neutral payload, replacement-request semantics, outer
retry budget, and sensitive-data boundary.
## Alternatives considered
- Continue blind module retries. This preserves a smaller contract but wastes
the module's deterministic diagnosis and commonly repeats the same defect.
- Convert module safety checks into validators. Typed reconciliation must apply
only safe proposal groups and retain a fallback before validation; moving
artifact-owned translation and application policy into validators would blur
stage ownership.
- Copy the retry reason or operator message into the model request. Those values
are written for provenance and humans, can contain opaque internal labels,
and do not reliably describe a correct replacement.
- Require feedback for every module retry. Structural failures may have no
valid exact candidate, so this would either prevent useful fresh retries or
fabricate prior-response material.
## Consequences
The normalize retry contract and generic producer-attempt directive gain an
optional correction-guidance field and candidate-pairing validation. Modules
that use it must provide semantically meaningful bounded prose and an exact
candidate. Registry reconciliation maintains separate operator and model
renderers, and policy fingerprints change so checkpoints created under blind
retry behavior are not reused.

View File

@@ -0,0 +1,59 @@
# ADR-0017: Defer cache access policy to the deployment
**Status:** Accepted
**Date:** 2026-09-11
## Context
Canonical chunk plans and invocation checkpoints are reconstructible cache
state. They may include source-derived content, so access and retention matter,
but Notarius cannot determine whether a deployment needs a private cache or a
cache shared by a trusted group. The current application-enforced `0700` and
`0600` modes prevent ordinary Unix provisioning from making that choice.
[ADR-0005](0005-cache-canonical-chunk-plans-by-source.md) governs canonical
chunk-plan reuse, while [ADR-0006](0006-separate-output-cache-and-debug-state.md)
established the output, cache, and debug separation refined by
[ADR-0007](0007-separate-checkpoint-recording-from-reuse.md). None of those
decisions requires Notarius to own exact cache access modes.
## Decision
Deployments own cache access policy through provisioned directories, process
umask, setgid inheritance, and default ACLs. Notarius requests conventional
maximum creation modes for cache entries and leaves the kernel to select their
effective permissions.
Notarius continues to own confined paths, symlink rejection, validation,
atomic publication, cleanup, and recovery. Those correctness boundaries are
independent from deciding which accounts may access cache state.
Source-derived cache contents can be sensitive. Operators own their access and
retention policy, while API credentials remain outside cache payloads under the
existing secret boundary. Existing cache inode modes are not migrated
automatically.
This decision does not supersede ADR-0005, ADR-0006, or ADR-0007.
## Alternatives considered
- Continue enforcing private `0700` directory and `0600` file modes. This
makes one deployment choice for every operator and prevents trusted-group
provisioning.
- Add configurable Notarius permission modes or a private/group cache setting.
This preserves application ownership of a deployment concern and expands the
public configuration surface without a durable policy reason.
- Recursively converge existing cache ownership, modes, or ACLs. This would
make upgrades mutate operator-managed state and cannot safely infer an
intended policy.
## Consequences
Operators can use a restrictive umask for private cache state or provision a
setgid/default-ACL cache root for trusted-group access. They must deliberately
recreate or administratively migrate pre-existing cache trees when their
existing modes do not match the desired deployment policy.
Notarius does not add user or group management, ACL mutation, permission
configuration, or recursive cache migration. Output and debug permission
contracts remain separate from this decision.

View File

@@ -0,0 +1,53 @@
# ADR-0018: Adopt ConfigKit for configuration loading
**Status:** Accepted
**Date:** 2026-09-13
## Context
Notarius's former configuration boundary owned strict YAML parsing, input
presence bookkeeping, defaults, normalization, environment overlays, and
configuration inspection. Those general-purpose responsibilities had grown
alongside application-specific pipeline and module semantics.
ConfigKit v0.3.0 provides strict typed YAML loading, imports, configuration
profiles, explicit CLI overrides, recorded defaults and normalization, safe
inspection rendering, and released source-origin APIs. The application needs
those source records to resolve filesystem paths according to the file that
supplied a surviving declaration.
## Decision
Notarius will adopt the root public `gitea.maximumdirect.net/eric/ConfigKit`
package at v0.3.0 for its version 5 configuration contract. ConfigKit owns
source composition, strict decoding, input presence, metadata, and safe
effective/explanation rendering. Notarius retains its typed document schema,
field semantics, runtime conversion, catalog-dependent pipeline resolution,
PromptKit profile handling, and physical run preparation.
Version 5 is a deliberate breaking file-contract change. It uses canonical
object module bindings and reference sources, and enables ConfigKit imports
and configuration profiles. The selected configuration file's canonical
directory is the base for its declared application paths; ConfigKit's released
`Resolution.InputInfo` and `Resolution.Source` APIs provide their origins.
`NOTARIUS_CONFIG` remains file discovery, while field-level operational
environment overlays are retired. Explicit configuration-valued CLI flags are
represented as ConfigKit overrides; invocation actions remain outside that
boundary.
## Alternatives considered
- Continue extending the custom YAML loader. This preserves duplicated
general-purpose parsing, presence, composition, and inspection machinery.
- Retain version 4 alongside version 5. This would leave a supported dual
configuration boundary and delay removal of the machinery being replaced.
- Wrap ConfigKit in a compatibility pre-parser. This weakens ConfigKit's
strict contract and recreates the application-owned parser complexity.
## Consequences
The released application has one ConfigKit-backed loader and one version 5
contract. Operator documentation and maintained examples move with the cutover.
ConfigKit metadata is retained only at the configuration publication boundary;
its types do not become module or runtime pipeline interfaces.

View File

@@ -12,8 +12,8 @@ defined in [Operations](operations.md).
notarius help notarius help
notarius --version notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags] notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate [--config path/to/config.yml] [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--json] notarius pipelines list [--config path/to/config.yml] [--config-profile name] [--json]
~~~ ~~~
Running Notarius without arguments, or with **help**, **--help**, or **-h**, Running Notarius without arguments, or with **help**, **--help**, or **-h**,
@@ -42,6 +42,7 @@ pipeline ID and **--input** are required.
| Flag | Meaning | | Flag | Meaning |
| --- | --- | | --- | --- |
| **--config path** | Use this configuration file. When omitted, configuration discovery applies; see [Configuration](config.md). | | **--config path** | Use this configuration file. When omitted, configuration discovery applies; see [Configuration](config.md). |
| **--config-profile name** | Select a ConfigKit configuration overlay. Empty and **default** select the composed base configuration. |
| **--input path** | Source input file to process. Required. | | **--input path** | Source input file to process. Required. |
| **--output-dir path** | Override the configured output root for this run. | | **--output-dir path** | Override the configured output root for this run. |
| **--json** | Write the successful run-result receipt as JSON to standard output. | | **--json** | Write the successful run-result receipt as JSON to standard output. |
@@ -55,8 +56,8 @@ pipeline ID and **--input** are required.
| **--session-id id** | Override the generated prompt session identifier with a non-empty value for LLM-backed module calls. | | **--session-id id** | Override the generated prompt session identifier with a non-empty value for LLM-backed module calls. |
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. | | **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. | | **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. | | **--reference selector=path** | Add or replace external file reference bindings at pipeline, lane, chunk, or binding scope. Repeatable. |
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. | | **--without-reference selector** | Remove matching configured external reference bindings. Repeatable. |
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**. **--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
**--debug-dir**, **--output-dir**, **--session-id**, and **--debug-dir**, **--output-dir**, **--session-id**, and
@@ -81,33 +82,62 @@ guidance.
### Reference selectors ### Reference selectors
Use **--reference** only for a reference slot declared by the selected Use **--reference** only for reference slots declared by the selected
configured target. The accepted selector forms are: configured targets. Qualification narrows the scope of an override:
| Form | Target | | Form | Target |
| --- | --- | | --- | --- |
| slot=path | The unique selected target that declares slot. | | slot=path | Every selected target that declares slot. |
| chunk.slot=path | The chunker. | | chunk.slot=path | The chunker. |
| merge.slot=path | The unique selected merger that declares slot. | | lane.slot=path | Every extractor, merger, or normalizer in lane that declares slot. |
| lane.slot=path | The unique extractor, merger, or normalizer in lane that declares slot. |
| lane.extract.slot=path | The extractor in lane. | | lane.extract.slot=path | The extractor in lane. |
| lane.merge.slot=path | The merger in lane. | | lane.merge.slot=path | The merger in lane. |
| lane.normalize.slot=path | The normalizer in lane. | | lane.normalize.slot=path | The normalizer in lane. |
**--without-reference** uses the same selector forms without =path. Slot Pipeline- and lane-scoped selectors are expected to match multiple targets and
names, requiredness, and configured bindings are part of the fail if they match none. A stage-specific selector fails when its lane is not
selected or its target does not declare the slot. There is no stage-wide
`merge.slot` shorthand; name the lane when targeting a merger.
CLI bindings override configured external paths. For overlapping CLI
selectors, a binding-specific or chunk selector overrides a lane selector, and
a lane selector overrides a pipeline selector. The last occurrence wins at
equal scope. Binding and unbinding the same concrete target at equal scope is
an error; a narrower bind or unbind may create an intentional exception to a
broader action.
**--without-reference** uses the same selector forms without `=path` and
removes external bindings only. Neither flag replaces or removes a generated
artifact handoff; an external/generated collision is a resolution error.
Required slots are checked after all effective changes. CLI reference paths
are resolved relative to the process working directory, so subprocess and
service callers should use absolute paths. Slot names, accepted media types,
size limits, requiredness, and configured generated bindings are part of the
[configuration contract](config.md). [configuration contract](config.md).
For example, one shared campaign reference can reach every compatible target,
with an optional lane-specific exception:
~~~
notarius run dnd-session \
--input /data/transcript.json \
--reference party=/data/references/party.txt \
--reference npc-registry.party=/data/references/npc-party-context.txt
~~~
### Run output ### Run output
Without **--json**, standard output contains the completed pipeline ID, counts Without **--json**, standard output contains the completed pipeline ID, counts
of normalized and rejected outputs, and the output directory. A debug-enabled of normalized and rejected outputs, and the output directory. A debug-enabled
run also prints its debug-bundle path to standard output. A successful run with run also prints its debug-bundle path to standard output. A successful run with
warnings reports the warning count to standard error. The published JSON bundle actionable process warnings reports their group and occurrence counts to
standard error. When the selected output module publishes `warnings.json`, the
summary also reports that durable file's path. Advisory and observation findings
do not produce a warning line. The published JSON bundle
is defined by the [JSON output contract](integrations/json-output.md). is defined by the [JSON output contract](integrations/json-output.md).
With **--json**, successful standard output is exactly one With **--json**, successful standard output is exactly one
`notarius.run-result.v1` JSON document followed by a newline, with no `notarius.run-result.v2` JSON document followed by a newline, with no
human-oriented status or debug-path line. Its fields and compatibility policy human-oriented status or debug-path line. Its fields and compatibility policy
are defined by the [run-result contract](integrations/run-result.md). A caller are defined by the [run-result contract](integrations/run-result.md). A caller
must check for exit status 0 before decoding this output; a failed write can must check for exit status 0 before decoding this output; a failed write can
@@ -125,10 +155,10 @@ OPENROUTER_API_KEY=your-api-key \
## config validate ## config validate
~~~ ~~~
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate [--config path/to/config.yml] [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
~~~ ~~~
This command loads and validates a configuration. With **--pipeline**, it also This command loads and validates the selected composed configuration. With **--pipeline**, it also
resolves that pipeline against the production module catalog. **--only** selects resolves that pipeline against the production module catalog. **--only** selects
lanes during that resolution and requires **--pipeline**. lanes during that resolution and requires **--pipeline**.
@@ -151,10 +181,32 @@ OPENROUTER_API_KEY=validation-placeholder \
The placeholder in the second command is sufficient only for offline The placeholder in the second command is sufficient only for offline
validation; it cannot run a provider-backed pipeline. validation; it cannot run a provider-backed pipeline.
## config effective
~~~
notarius config effective [--config path/to/config.yml] [--config-profile name]
~~~
This command writes the selected composed configuration as deterministic,
redacted YAML. Standard output contains only that YAML: it does not validate a
pipeline, prepare modules, materialize references, or contact a provider.
## config explain
~~~
notarius config explain [--config path/to/config.yml] [--config-profile name] <json-pointer>
~~~
This command writes ConfigKit's redacted, human-readable explanation for one
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
pointer is a syntax error; a well-formed unknown path is a command failure.
The explanation can describe a value displaced by a selected profile. Standard
output contains only the explanation.
## pipelines list ## pipelines list
~~~ ~~~
notarius pipelines list [--config path/to/config.yml] [--json] notarius pipelines list [--config path/to/config.yml] [--config-profile name] [--json]
~~~ ~~~
This command lists configured pipeline IDs in sorted order. By default, it This command lists configured pipeline IDs in sorted order. By default, it
@@ -171,7 +223,11 @@ go run ./cmd/notarius pipelines list \
Successful commands write their primary result to standard output. Warnings and Successful commands write their primary result to standard output. Warnings and
errors are written to standard error. errors are written to standard error.
For **run --json**, warnings remain on standard error and standard output is a Configuration load errors include ConfigKit's field diagnostics and source
coordinates, followed by a source-ID legend identifying the acquired files or
source labels. This includes imported files and the selected configuration profile.
For **run --json**, actionable process warnings remain on standard error and standard output is a
machine-readable success result only. Syntax and runtime diagnostics remain on machine-readable success result only. Syntax and runtime diagnostics remain on
standard error. Parse the result only after the process exits with status 0. standard error. Parse the result only after the process exits with status 0.
@@ -182,6 +238,6 @@ standard error. Parse the result only after the process exits with status 0.
| 2 | The command or flag syntax was invalid, including unknown commands, missing required arguments, invalid flag values, or invalid flag combinations. | | 2 | The command or flag syntax was invalid, including unknown commands, missing required arguments, invalid flag values, or invalid flag combinations. |
The root help spellings are the supported help path. Invoking **--help** on The root help spellings are the supported help path. Invoking **--help** on
**run**, **config validate**, or **pipelines list** is handled by the flag **run**, a **config** subcommand, or **pipelines list** is handled by the flag
parser as a usage error: it writes an error to standard error and exits with parser as a usage error: it writes an error to standard error and exits with
status 2. status 2.

View File

@@ -1,7 +1,7 @@
# Configuration # Configuration
This is the canonical reference for Notarius configuration. Configuration files This is the canonical reference for Notarius configuration. Configuration files
are YAML and must declare version 4. They select pipelines and their modules; are YAML and must declare version 5. They select pipelines and their modules;
the [CLI reference](cli.md) owns invocation syntax, and the [CLI reference](cli.md) owns invocation syntax, and
[Operations](operations.md) owns run-state procedures. [Operations](operations.md) owns run-state procedures.
@@ -19,12 +19,15 @@ The command fails if none of these paths provides a configuration file.
For configuration values, precedence is: For configuration values, precedence is:
1. built-in defaults; 1. built-in defaults;
2. the selected YAML file; 2. composed YAML sources, including imports;
3. supported operational environment variables; and 3. the selected ConfigKit profile; and
4. the CLI run overrides that apply to a command. 4. explicitly supplied configuration-valued run flags.
Environment variables do not provide a second configuration schema. They only `--config-profile` selects a ConfigKit overlay from `profiles/<name>.yml`
override the fields listed below. beneath the selected configuration root. Empty and `default` select no overlay.
Configuration profiles are distinct from PromptKit LLM profiles selected by
`llm_profile` or `--llm-profile`. The retired operational environment variables
have no effect; `NOTARIUS_CONFIG` remains a source-discovery input.
## Maintained Examples ## Maintained Examples
@@ -33,7 +36,8 @@ override the fields listed below.
- [Complete D&D configuration](../examples/dnd-complete.config.yml) uses - [Complete D&D configuration](../examples/dnd-complete.config.yml) uses
ordered steps, all implemented D&D lanes, generated references, state ordered steps, all implemented D&D lanes, generated references, state
settings, bounded LLM concurrency, and the maintained settings, bounded LLM concurrency, and the maintained
[operator profile](../examples/profiles/dnd-extraction.yml). [LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
pipeline from [a fragment](../examples/pipelines/dnd-session.yml).
Use these complete files as starting points rather than combining the Use these complete files as starting points rather than combining the
illustrative fragments in this reference. illustrative fragments in this reference.
@@ -46,7 +50,7 @@ other than **version** is optional.
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| **version** | integer | none | Required; must be 4. | | **version** | integer or quoted string | none | Required; must be 5. Maintained files use `version: 5`. |
| **promptkit** | object | none | Profile source and optional local-backend configuration. | | **promptkit** | object | none | Profile source and optional local-backend configuration. |
| **pipelines** | map | empty | Maps pipeline IDs to pipeline definitions. | | **pipelines** | map | empty | Maps pipeline IDs to pipeline definitions. |
| **concurrency** | object | see below | Global LLM and extraction limits. | | **concurrency** | object | see below | Global LLM and extraction limits. |
@@ -69,6 +73,22 @@ Built-in defaults are:
An empty cache directory in YAML deliberately selects the corresponding An empty cache directory in YAML deliberately selects the corresponding
per-user root. An explicit empty output or debug directory is invalid. per-user root. An explicit empty output or debug directory is invalid.
Selecting a chunk-plan or checkpoint directory does not configure filesystem
access policy. Cache permissions come from deployment provisioning, process
umask, setgid inheritance, and default ACLs; see [Operations](operations.md#cache-provisioning).
Every nonempty relative configured destination or reference path resolves from
the canonical directory of the file that supplies its surviving declaration.
This includes imported fragments and selected configuration profiles. Built-in
relative destinations use the selected main configuration file's directory;
explicit `--output-dir` and `--debug-dir` values use the invocation working
directory. Absolute values remain unchanged.
The root file can use ConfigKit imports to compose disjoint definitions. Imports
and selected profiles remain under the configuration root; they do not merge or
append a pipeline's `steps` sequence. A profile replaces sequences as defined
by ConfigKit, so changing an individual ordered binding requires supplying the
whole replacement sequence.
## PromptKit Profiles ## PromptKit Profiles
@@ -76,7 +96,7 @@ The optional **promptkit** object selects one source of profile definitions and
may register one conventional local OpenAI-compatible backend: may register one conventional local OpenAI-compatible backend:
~~~yaml ~~~yaml
version: 4 version: 5
promptkit: promptkit:
profile_dir: ./profiles profile_dir: ./profiles
@@ -94,17 +114,18 @@ promptkit:
| **local_backend.endpoint** | string | Required when **local_backend** is present; absolute HTTP or HTTPS URL with a host. | | **local_backend.endpoint** | string | Required when **local_backend** is present; absolute HTTP or HTTPS URL with a host. |
| **local_backend.concurrency_limit** | integer | Optional non-negative limit; defaults to 0. | | **local_backend.concurrency_limit** | integer | Optional non-negative limit; defaults to 0. |
Set at most one of **profile_dir** and **profile_file**. Relative values use Set at most one of **profile_dir** and **profile_file**. Every nonempty relative
the process working directory, not the configuration file's directory. The configured filesystem path is resolved from the canonical directory of the
complete example's `./examples/profiles/dnd-extraction.yml` value is therefore file that supplies it. The complete example's
valid when Notarius is launched from the repository root; use an absolute path `./llm-profiles/dnd-extraction.yml` value therefore resolves beside that example.
for services and containers. Use absolute paths for services and containers when their deployment layout is
not configuration-relative.
An operator source is optional. For a requested ID, PromptKit checks the An operator source is optional. For a requested ID, PromptKit checks the
configured operator source first, then Notarius's embedded fallback profiles, configured operator source first, then Notarius's embedded fallback profiles,
then its own built-in catalog. A matching profile is complete: it replaces a then its own built-in catalog. A matching profile is complete: it replaces a
lower-precedence definition rather than merging with it. The maintained lower-precedence definition rather than merging with it. The maintained
[`dnd-extraction` operator profile](../examples/profiles/dnd-extraction.yml) [`dnd-extraction` LLM profile](../examples/llm-profiles/dnd-extraction.yml)
is a secret-free deployment artifact; production, development, and local is a secret-free deployment artifact; production, development, and local
deployments can each provide a complete definition with that same workload ID. deployments can each provide a complete definition with that same workload ID.
Use workload-oriented IDs for new profiles instead of model names. Use workload-oriented IDs for new profiles instead of model names.
@@ -132,7 +153,7 @@ model: example-model
Keep credentials out of the local-backend object. A PromptKit profile may name Keep credentials out of the local-backend object. A PromptKit profile may name
its credential environment variable through `api_key_env`; set that variable its credential environment variable through `api_key_env`; set that variable
only in the run environment. PromptKit owns the only in the run environment. PromptKit owns the
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md), [pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
including `base_profile` inheritance. Notarius passes profiles through without including `base_profile` inheritance. Notarius passes profiles through without
merging them. Filesystem profiles cannot express PromptKit's in-memory merging them. Filesystem profiles cannot express PromptKit's in-memory
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the `APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
@@ -147,30 +168,15 @@ requiring credential values. It rejects absent, malformed, or incompatible
profiles before a run prepares modules. Credential availability is checked only profiles before a run prepares modules. Credential availability is checked only
when a generation is prepared. when a generation is prepared.
## Migrating Version 3 Configuration ## Version 5 Migration
Version 3 files are not decoded or rewritten. Change **version: 3** to Version 3 and version 4 files are rejected and are not rewritten at load time.
**version: 4** and rename the top-level **scriptorium:** section to Use `version: 5`, rename the former `scriptorium` section to `promptkit`, and
**promptkit:**. Version 4 decoding is strict, so a remaining **scriptorium** convert every module binding to an object containing `module`. Convert every
field is rejected as unknown. external reference to `{path: ...}` and move any lane-level references to the
corresponding extraction binding. Put durable operational values in YAML or
## Operational Environment Variables use the documented explicit run flags; the former field-level environment
variables are not read.
These variables are applied after YAML values:
| Variable | Overrides | Rules |
| --- | --- | --- |
| **NOTARIUS_TOTAL_LLM_CONCURRENCY** | **concurrency.total_llm** | Integer. |
| **NOTARIUS_STAGE_WORKERS_EXTRACT** | **concurrency.stage_workers.extract** | Integer. |
| **NOTARIUS_OUTPUT_DIR** | **output.directory** | Non-empty path. |
| **NOTARIUS_CACHE_CHUNK_PLANS_MODE** | **cache.chunk_plans.mode** | **auto**, **bypass**, or **refresh**. |
| **NOTARIUS_CACHE_CHUNK_PLANS_DIR** | **cache.chunk_plans.directory** | Non-empty path. |
| **NOTARIUS_CACHE_CHECKPOINTS_DIR** | **cache.checkpoints.directory** | Non-empty path. |
| **NOTARIUS_DEBUG_DIR** | **debug.directory** | Non-empty path. |
Integer values are trimmed then parsed as base-10 integers. Directory and
output values reject NUL characters. **NOTARIUS_CONFIG** participates only in
configuration discovery.
## Concurrency, Output, Cache, And Debug ## Concurrency, Output, Cache, And Debug
@@ -195,11 +201,14 @@ debug:
**concurrency.total_llm** must be greater than zero. The only supported **concurrency.total_llm** must be greater than zero. The only supported
**concurrency.stage_workers** key is **extract**; its value must be from 1 **concurrency.stage_workers** key is **extract**; its value must be from 1
through **total_llm**. When omitted, it is recalculated from the effective through **total_llm**. When omitted, it is recalculated from the effective
**total_llm** after YAML and environment precedence. **total_llm** after imports, profile selection, and explicit ConfigKit
overrides.
**cache.chunk_plans.mode** accepts **auto**, **bypass**, or **refresh**. **cache.chunk_plans.mode** accepts **auto**, **bypass**, or **refresh**.
**cache.checkpoints.enabled** is a boolean. The CLI can override the output **cache.checkpoints.enabled** is a boolean. The CLI can override the output
directory and chunk-plan mode for one run; see [CLI reference](cli.md#run). directory and chunk-plan mode for one run; see [CLI reference](cli.md#run).
Cache directory fields select only locations; Notarius has no cache permission
field or operational environment override.
## Pipelines ## Pipelines
@@ -209,20 +218,27 @@ Each **pipelines** entry has a unique, non-empty ID and the following shape:
pipelines: pipelines:
dnd-session: dnd-session:
llm_profile: dnd-extraction llm_profile: dnd-extraction
input: seriatim input:
chunk: generic module: seriatim
output: json chunk:
module: generic
output:
module: json
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
merge: appendorder module: dnd/spells
normalize: dnd/spells merge:
module: appendorder
normalize:
module: dnd/spells
~~~ ~~~
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. | | **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. | | **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
| **input** | module binding | none | Required. | | **input** | module binding | none | Required. |
| **chunk** | module binding | **generic** | Optional. | | **chunk** | module binding | **generic** | Optional. |
| **output** | module binding | **json** | Optional. | | **output** | module binding | **json** | Optional. |
@@ -251,6 +267,33 @@ explicit `null` and non-integer values are invalid. An explicit value on a
deterministic binding or validator is invalid, while a pipeline value simply deterministic binding or validator is invalid, while a pipeline value simply
does not apply to deterministic selections. does not apply to deterministic selections.
`validation_policy` controls terminal disposition for one complete producer
attempt and validator chain. It may appear on a pipeline or a **chunk**,
**extract**, **merge**, or **normalize** module binding; input, output, and
validator bindings reject it. Every field is optional and resolves in binding,
pipeline, then application-default order:
| Field | Values | Default |
| --- | --- | --- |
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
The policy object and its fields must be non-null, and unknown fields are
rejected. A deterministic producer may not explicitly set
**producer_structural_failure** on its binding, although a pipeline-level
default remains valid for pipelines that include LLM-backed producers.
After the producer binding's retry budget is exhausted, an invalid structured
response uses **producer_structural_failure**. One or more semantic validator
rejections use **semantic_rejection**; rejection takes precedence over an
exhausted validator failure or skip. With no rejection, an exhausted validator
failure or skip uses **validator_failure**. `reject_output` records the
terminal rejection without advancing that candidate. `warn_continue` is valid
only for validator execution failure: it advances a structurally valid,
otherwise unrejected result with incomplete-validation provenance and without
making it reusable checkpoint state.
A lane has these fields: A lane has these fields:
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
@@ -258,19 +301,20 @@ A lane has these fields:
| **extract** | module binding | none | Required. | | **extract** | module binding | none | Required. |
| **merge** | module binding | **appendorder** | Optional. | | **merge** | module binding | **appendorder** | Optional. |
| **normalize** | module binding | **noop** | Optional. | | **normalize** | module binding | **noop** | Optional. |
| **references** | map | none | Supported compatibility alias for **extract.references**. | | **references** | map | none | Not supported at lane scope; place references on the eligible binding. |
| **validators** | list | none | Non-empty lane-level lists are rejected. Set validator overrides on a binding instead. | | **validators** | list | none | Non-empty lane-level lists are rejected. Set validator overrides on a binding instead. |
The lane-level **references** alias remains accepted. When the alias and Lane-level **references** are not supported. Put a lane extraction reference
**extract.references** bind the same slot, **extract.references** wins. Use under **extract.references**.
the binding-local form in new configurations.
## Module Bindings And Validators ## Module Bindings And Validators
Use a module key directly when no other binding fields are needed: Every module and validator binding uses an object, even when it has no fields
beyond its module key:
~~~yaml ~~~yaml
input: seriatim input:
module: seriatim
~~~ ~~~
Use an object for fields: Use an object for fields:
@@ -281,7 +325,8 @@ extract:
llm_profile: dnd-extraction llm_profile: dnd-extraction
retries: 2 retries: 2
references: references:
spell_catalog: ./dnd-spell-catalog.json spell_catalog:
path: ./dnd-spell-catalog.json
~~~ ~~~
| Binding field | Type | Default | Rules | | Binding field | Type | Default | Rules |
@@ -289,7 +334,8 @@ extract:
| **module** | string | none | Required for an object binding. Must be a registered compatible key. | | **module** | string | none | Required for an object binding. Must be a registered compatible key. |
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. | | **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. | | **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. | | **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
| **options** | object | none | Must satisfy the selected module. | | **options** | object | none | Must satisfy the selected module. |
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. | | **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. | | **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
@@ -297,11 +343,23 @@ extract:
Omitting **validators** uses the registered chain. **validators: []** selects Omitting **validators** uses the registered chain. **validators: []** selects
an empty chain; a non-empty list replaces the chain in the listed order. an empty chain; a non-empty list replaces the chain in the listed order.
Validator bindings accept only **module**, **llm_profile**, Validator bindings accept only **module**, **llm_profile**,
**structured_output_repair_attempts**, and **options**. They reject **structured_output_repair_attempts**, **retries**, and **options**. Their
**references**, **retries**, and nested **validators**. Deterministic validators **retries** value is a non-negative additional validator-execution budget and
reject explicit **llm_profile** and **structured_output_repair_attempts**. is valid only when the selected validator is LLM-backed. A validator retry
rechecks the same immutable candidate; it never regenerates the producer.
They reject
**validation_policy**, **references**, and nested **validators**. Deterministic
validators reject explicit **llm_profile** and
**structured_output_repair_attempts**.
Deterministic module bindings also reject those explicit fields. Deterministic module bindings also reject those explicit fields.
An LLM-backed chunk, extract, merge, or normalize producer with both a
non-empty validator chain and positive **retries** must declare the supported
single-response correction capability. Preparation rejects a configuration
that could require semantic correction from a producer that cannot provide an
exact prior response. A deterministic producer, or an LLM attempt that did
not make a model call, cannot consume a semantic retry after rejection.
The **json** output module accepts optional **include_chunk_map** and The **json** output module accepts optional **include_chunk_map** and
**evidence_context** settings: **evidence_context** settings:
@@ -340,18 +398,20 @@ When enabled, it publishes the selected source-unit excerpt defined by the
## References And Ordered Handoffs ## References And Ordered Handoffs
Reference maps bind named slots that the selected target declares. A scalar is Reference maps bind named slots that the selected target declares. An external
an external path. Pipeline-level maps accept only external paths; step-local reference is an object with `path`. Pipeline-level maps accept only external
and binding-local maps may also select a normalized artifact from an earlier paths; step-local and binding-local maps may also select a normalized artifact
step: from an earlier step:
~~~yaml ~~~yaml
steps: steps:
- id: describe-session - id: describe-session
artifacts: artifacts:
npc-registry: npc-registry:
extract: dnd/npc-registry extract:
normalize: dnd/npc-registry module: dnd/npc-registry
normalize:
module: dnd/npc-registry
- id: extract-events - id: extract-events
references: references:
npc_registry: npc_registry:
@@ -360,8 +420,10 @@ steps:
lane: npc-registry lane: npc-registry
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
normalize: dnd/spells module: dnd/spells
normalize:
module: dnd/spells
~~~ ~~~
An artifact selector contains only **step** and **lane**. The producer must be An artifact selector contains only **step** and **lane**. The producer must be
@@ -370,10 +432,16 @@ slot. A generated binding supplies one accepted normalized artifact; it does
not name a file. A configured generated dependency remains required even when not name a file. A configured generated dependency remains required even when
that consumer slot is otherwise optional. that consumer slot is otherwise optional.
Pipeline references are defaults. A matching step-local or binding-local Pipeline references are configuration defaults. A matching step-local or
external path overrides a pipeline default. Required slots must be bound after binding-local external path overrides a pipeline default. CLI reference
these configuration values and any CLI reference overrides are applied. bindings are then operational overrides of configured external paths; their
Reference paths in YAML are resolved relative to the configuration file. pipeline, lane, and binding scopes and precedence are defined by the
[CLI reference](cli.md#reference-selectors). A CLI file reference cannot
replace a configured generated artifact handoff. Required slots must be bound
after configuration and CLI reference actions are applied. A YAML reference
path resolves from the file that supplies its surviving declaration, including
an imported fragment or selected profile. CLI reference paths resolve relative
to the process working directory.
### D&D Reference Slots ### D&D Reference Slots
@@ -463,11 +531,33 @@ Available validator keys are:
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** | | Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** | | Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** | | NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** | | Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** |
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** | | Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** | | Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** | | Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene:
~~~yaml
extract:
module: dnd/scene-descriptions
retries: 1
validators:
- module: generic/valid_json
- module: extract/dnd/scene-descriptions/shape
- module: extract/dnd/scene-descriptions/source_refs
- module: generic/valid_json_schema
- module: extract/dnd/scene-descriptions/source_relatedness
- module: extract/dnd/scene-descriptions/combat_semantics
retries: 1
~~~
An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules.
The validator retry shown above permits one additional execution against the
same scene candidate when the LLM-backed validator itself fails; it is separate
from both the extractor's producer retry and PromptKit structural repair.
When no override is configured, production D&D bindings use the following When no override is configured, production D&D bindings use the following
ordered chains. Each row lists extract then normalize; spell chains are the ordered chains. Each row lists extract then normalize; spell chains are the
same at both stages. same at both stages.
@@ -500,7 +590,16 @@ go run ./cmd/notarius config validate \
~~~ ~~~
Configuration validation rejects invalid YAML, unsupported fields, invalid Configuration validation rejects invalid YAML, unsupported fields, invalid
defaults or environment overrides, incompatible module keys, unknown options, defaults or explicit CLI overrides, incompatible module keys, unknown options,
invalid reference bindings, missing required reference slots, invalid validator invalid reference bindings, missing required reference slots, invalid validator
overrides, and incompatible generated artifact handoffs. Use overrides, and incompatible generated artifact handoffs. Use
[pipelines list](cli.md#pipelines-list) to inspect configured IDs. [pipelines list](cli.md#pipelines-list) to inspect configured IDs.
## Inspection
Use [config effective](cli.md#config-effective) to write the selected composed
configuration as deterministic, redacted YAML. Use
[config explain](cli.md#config-explain) with a canonical JSON Pointer to inspect
the effective or historical provenance for one path, including a value displaced
by a selected profile. Neither command resolves a pipeline, prepares modules,
materializes references, or requires provider credentials.

View File

@@ -23,14 +23,14 @@ A deployment must provide its own PromptKit profile and campaign reference
files. Use absolute paths for service and subprocess deployments. In files. Use absolute paths for service and subprocess deployments. In
particular, observe these different resolution rules: particular, observe these different resolution rules:
- reference paths in YAML are resolved relative to the Notarius configuration - reference and PromptKit profile paths in YAML are resolved relative to the
file; and file that supplies their surviving declaration;
- `promptkit.profile_file` is resolved relative to the Notarius process working - reference paths passed with `--reference` are resolved relative to the
directory. Notarius process working directory.
Do not copy the repository example's relative profile path into a deployment Do not copy the repository example's relative profile path into a deployment
without also controlling that working directory. The complete path and profile without preserving its configuration-relative layout. The complete path and
rules are defined in [Configuration](../config.md). profile rules are defined in [Configuration](../config.md).
Preflight the deployed configuration before processing sessions and whenever Preflight the deployed configuration before processing sessions and whenever
it changes: it changes:
@@ -73,9 +73,21 @@ notarius run dnd-session \
--config /absolute/path/to/notarius.yml \ --config /absolute/path/to/notarius.yml \
--input /absolute/path/to/transcripts/final.trimmed.json \ --input /absolute/path/to/transcripts/final.trimmed.json \
--output-dir /absolute/path/to/notarius-output \ --output-dir /absolute/path/to/notarius-output \
--reference party=/absolute/path/to/references/party.txt \
--reference players=/absolute/path/to/references/players.txt \
--reference glossary=/absolute/path/to/references/glossary.txt \
--reference spell_catalog=/absolute/path/to/references/spells.json \
--json --json
``` ```
Each unqualified reference is pipeline-scoped: Notarius supplies it to every
selected D&D target that declares the slot. A deployment may omit an optional
reference it does not maintain, and may use the lane- or binding-qualified
forms from the [CLI reference](../cli.md#reference-selectors) for an exceptional
override. The registry, scene-description, combat-turn, and NPC-occurrence
references declared between ordered steps in the complete configuration are
generated artifacts. Do not pass those handoffs on the CLI.
The caller should: The caller should:
- capture stdout and stderr separately; - capture stdout and stderr separately;
@@ -92,7 +104,7 @@ stream and exit-status contract.
## Discover The Published Bundle ## Discover The Published Bundle
Decode the successful stdout document as a supported run-result schema. For Decode the successful stdout document as a supported run-result schema. For
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate
unknown fields allowed by that version, but reject an unsupported schema unknown fields allowed by that version, but reject an unsupported schema
version. version.
@@ -101,6 +113,12 @@ root. Do not scan the output root for its newest directory, guess a run ID, or
construct a bundle path. Resolve `index_file` beneath `output_directory` and construct a bundle path. Resolve `index_file` beneath `output_directory` and
reject an absolute logical path or any result that escapes the bundle root. reject an absolute logical path or any result that escapes the bundle root.
The complete configuration uses the application validation defaults. A caller
that requires fully validated D&D artifacts must also require receipt
`validation_status: approved`; a successful `incomplete` result reflects the
configured validator-failure continuation policy and carries its bounded
validator provenance in `validation_summaries`.
Read `index.json` and locate each requested lane in `output_files` by its exact Read `index.json` and locate each requested lane in `output_files` by its exact
`lane_id`. Do not guess a lane filename. Before decoding a payload: `lane_id`. Do not guess a lane filename. Before decoding a payload:
@@ -139,7 +157,8 @@ The JSON encoder always publishes these bundle-management files:
| `index.json` | Discovery document for lane and pipeline-wide artifacts. | | `index.json` | Discovery document for lane and pipeline-wide artifacts. |
| `manifest.json` | Run provenance and result summaries. | | `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. | | `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. | | `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Advisory and observation findings for accepted artifacts. |
The complete configuration also requests two pipeline-wide artifacts: The complete configuration also requests two pipeline-wide artifacts:

View File

@@ -15,9 +15,12 @@ configured lane inventory, and downstream acceptance checklist, see
Optionally preflight a selected configuration and pipeline before work starts: Optionally preflight a selected configuration and pipeline before work starts:
```sh ```sh
notarius config validate --config /path/to/notarius.yml --pipeline pipeline-id notarius config validate --config /path/to/notarius.yml --config-profile production --pipeline pipeline-id
``` ```
Omit `--config-profile` when the composed base configuration is the intended
selection; it is independent from any `--llm-profile` run override.
Invoke the run with explicit paths and machine-readable output. Capture Invoke the run with explicit paths and machine-readable output. Capture
standard output and standard error separately; do not combine them before standard output and standard error separately; do not combine them before
processing the result. processing the result.
@@ -25,19 +28,43 @@ processing the result.
```sh ```sh
notarius run pipeline-id \ notarius run pipeline-id \
--config /path/to/notarius.yml \ --config /path/to/notarius.yml \
--config-profile production \
--input /path/to/source.json \ --input /path/to/source.json \
--output-dir /path/to/output-root \ --output-dir /path/to/output-root \
--json --json
``` ```
Use absolute paths for supplied input, configuration, output-root, and Use absolute paths for supplied input, configuration, output-root, and
reference files. Notarius generates a stable prompt session for the resolved reference files. Pass each external reference as its own argument-vector pair;
input module and exact input bytes. Pass **--session-id** only when intentionally do not construct and invoke a shell command. An unqualified reference selector
grouping different invocations under a different session. Supply credentials supplies that file to every compatible selected target. Lane and stage
through Notarius's documented configuration and environment mechanisms, never qualification are available for exceptional overrides, while generated
as command-line arguments or generated secret-bearing configuration. In same-run references remain part of configured pipeline composition. The
particular, a session identifier is provider-visible and is not a credential [CLI reference](../cli.md#reference-selectors) owns the exact selector and
mechanism. precedence contract.
The maintained D&D subprocess workflow uses this facility for campaign context:
```sh
notarius run dnd-session \
--config /absolute/path/to/notarius.yml \
--input /absolute/path/to/transcripts/final.trimmed.json \
--output-dir /absolute/path/to/notarius-output \
--reference party=/absolute/path/to/references/party.txt \
--reference players=/absolute/path/to/references/players.txt \
--reference glossary=/absolute/path/to/references/glossary.txt \
--reference spell_catalog=/absolute/path/to/references/spells.json \
--json
```
Only pass the external references available to and desired by the deployment.
Notarius generates a stable prompt session for the resolved input module and
exact input bytes; reference changes do not change it. Pass **--session-id**
only when intentionally grouping different invocations under a different
session. Supply credentials through Notarius's documented configuration and
environment mechanisms, never as command-line arguments or generated
secret-bearing configuration. In particular, a session identifier is
provider-visible and is not a credential mechanism.
Wait for the process before interpreting standard output. Only an exit status Wait for the process before interpreting standard output. Only an exit status
of 0 permits decoding the receipt. On a nonzero exit, retain standard error for of 0 permits decoding the receipt. On a nonzero exit, retain standard error for
@@ -69,10 +96,17 @@ lanes. The caller decides which lane IDs are required for its own work and
which are optional; it should make that decision explicitly rather than infer which are optional; it should make that decision explicitly rather than infer
failure from the receipt counts alone. failure from the receipt counts alone.
When complete validation is required, also require receipt
`validation_status: approved` and inspect `validation_summaries`. A successful
run with `validation_status: incomplete` contains a structurally valid result
that advanced after validator execution could not complete under the configured
`warn_continue` policy. It is not reusable checkpoint state and should not be
silently treated as fully reviewed by the caller.
## Preserve Provenance And Handle Data Carefully ## Preserve Provenance And Handle Data Carefully
Keep the receipt with the published `manifest.json`, and retain Keep the receipt with the published `manifest.json`, and retain
`rejected.json` and `warnings.json` when review or later provenance requires `rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires
them. Treat the input, output bundle, cache, debug bundle, and captured process them. Treat the input, output bundle, cache, debug bundle, and captured process
logs as potentially sensitive data. Apply the caller's access controls and logs as potentially sensitive data. Apply the caller's access controls and
retention policy, and avoid copying secrets into arguments, logs, or retention policy, and avoid copying secrets into arguments, logs, or

View File

@@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo
## Bundle Layout ## Bundle Layout
All paths below are logical, relative, slash-separated bundle paths. The All paths below are logical, relative, slash-separated bundle paths. The
encoder always emits the first four JSON files below and adds lane or encoder always emits the first five JSON files below and adds lane or
pipeline-wide artifact files when their corresponding artifacts are available: pipeline-wide artifact files when their corresponding artifacts are available:
A subprocess caller first obtains the physical bundle root from the A subprocess caller first obtains the physical bundle root from the
@@ -21,7 +21,8 @@ root for the logical discovery described here.
| `index.json` | Entry point that names the other published files and lane payloads. | | `index.json` | Entry point that names the other published files and lane payloads. |
| `manifest.json` | Run provenance and result summaries. | | `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. | | `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. | | `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. |
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. | | `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. | | `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. | | `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
@@ -39,7 +40,8 @@ normalized lanes has this valid minimal index:
"manifest_file": "manifest.json", "manifest_file": "manifest.json",
"output_files": [], "output_files": [],
"rejected_file": "rejected.json", "rejected_file": "rejected.json",
"warnings_file": "warnings.json" "warnings_file": "warnings.json",
"diagnostics_file": "diagnostics.json"
} }
``` ```
@@ -49,6 +51,7 @@ normalized lanes has this valid minimal index:
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. | | `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
| `rejected_file` | Yes | Always `rejected.json`. | | `rejected_file` | Yes | Always `rejected.json`. |
| `warnings_file` | Yes | Always `warnings.json`. | | `warnings_file` | Yes | Always `warnings.json`. |
| `diagnostics_file` | Yes | Always `diagnostics.json`. |
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. | | `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. | | `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
@@ -92,7 +95,7 @@ group into the following externally observable summaries:
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` | | Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` | | Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
| Source and references | `source_digests`, `references` | | Source and references | `source_digests`, `references` |
| Published result summaries | `normalized_outputs`, `rejected_outputs` | | Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` | | Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
`references` records provenance such as the target, slot, origin, digest, `references` records provenance such as the target, slot, origin, digest,
@@ -102,6 +105,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
provenance for the plan used by this run; cache records, debug artifacts, and provenance for the plan used by this run; cache records, debug artifacts, and
other operational state are not published as bundle files. other operational state are not published as bundle files.
Each `validation_summaries` entry is a bounded outcome for one producer result.
It has required `status`, `producer_attempt_count`, and `terminal_action`;
the stage and affected step, lane, module, or chunk identity are present when
applicable. `status` is `complete`, `rejected`, or `incomplete`.
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
configured validator order and omit later duplicates. Entries contain no raw
candidate response, correction guidance, validator diagnostic message, or
artifact payload. The same shape may appear as `validation` on an affected
rejection entry.
When present, `metadata.session_id` is the effective non-secret routing When present, `metadata.session_id` is the effective non-secret routing
correlation identifier used for the run. It can be visible to providers and is correlation identifier used for the run. It can be visible to providers and is
not a substitute for a cache or checkpoint identity. Its generation and not a substitute for a cache or checkpoint identity. Its generation and
@@ -122,16 +135,57 @@ These values describe observed execution; they are not a backend-registration
interface. Entries that differ by backend or effective reasoning remain interface. Entries that differ by backend or effective reasoning remain
distinct even when their profile, provider, and model are otherwise equal. distinct even when their profile, provider, and model are otherwise equal.
## Rejections And Warnings ## Rejections, Warnings, And Diagnostics
`rejected.json` is always an object with a `rejected` array. Each entry has `rejected.json` is always an object with a `rejected` array. Each entry has
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`, required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and `chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` are present only when applicable. `diagnostic_artifact_path` are present only when applicable. An entry may also
contain the bounded `validation` summary described above; the existing singular
validator and reason fields remain the first configured rejection for
compatibility.
`warnings.json` is always an object with a `warnings` array. Each warning has `warnings.json` is always the `notarius.warnings.v2` envelope:
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
there is nothing to report. ```json
{
"schema_version": "notarius.warnings.v2",
"group_count": 0,
"occurrence_count": 0,
"groups": []
}
```
It contains only process warnings. `group_count` is exact, and
`occurrence_count` is the exact sum of its group occurrence counts.
`diagnostics.json` is always the `notarius.diagnostics.v1` envelope:
```json
{
"schema_version": "notarius.diagnostics.v1",
"group_count": 0,
"occurrence_count": 0,
"truncated": false,
"unrepresented_occurrence_count": 0,
"groups": []
}
```
It contains only advisory and observation groups. `group_count` counts groups
represented in `groups`; `occurrence_count` includes both represented and
unrepresented occurrences. When `truncated` is true,
`unrepresented_occurrence_count` is the exact number omitted from group
representation.
Each group has `disposition`, `category`, `reason_code`, framework-owned
`origin`, exact `occurrence_count`, bounded `samples`, and
`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk
ID and zero-based chunk index when applicable. A group retains at most three
distinct samples. The framework fails rather than truncating actionable
warnings beyond 128 groups; it represents at most 256 advisory/observation
groups and records further occurrences through the diagnostic truncation
fields above.
## Compatibility ## Compatibility

View File

@@ -0,0 +1,13 @@
# ConfigKit Integration
Notarius uses the public root package
[`gitea.maximumdirect.net/eric/ConfigKit`](https://gitea.maximumdirect.net/eric/ConfigKit)
at v0.3.0 for version 5 configuration acquisition, imports, profiles, explicit
overrides, diagnostics, and safe inspection rendering.
Notarius owns the typed document, domain defaults and validation, runtime
conversion, catalog-dependent pipeline resolution, source-aware configured
paths, and command discovery. It uses only ConfigKit's public path, result,
diagnostic, and resolution-source APIs. Upstream documentation owns general
ConfigKit semantics; [Configuration](../config.md) owns Notarius's file and
operator contract.

View File

@@ -1,11 +1,11 @@
# PromptKit Integration # PromptKit Integration
Notarius pins Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0) [`gitea.maximumdirect.net/eric/promptkit` v0.10.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0)
as its in-process prompt engine. The upstream as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/consumers/pkg-promptkit.md) [Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md) [format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0/docs/formats.md)
owns prompt, profile, and schema file contracts. owns prompt, profile, and schema file contracts.
## Supported Boundary ## Supported Boundary
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
- construct an `Engine` with filesystem-backed prompt, schema, and optional - construct an `Engine` with filesystem-backed prompt, schema, and optional
operator and application-fallback profile sources; operator and application-fallback profile sources;
- prepare one frozen execution from a `RunRequest` with named inline artifacts, - prepare one frozen execution from a `RunRequest` with named inline artifacts,
variables, a direct session ID, prompt identity, and profile selection, then variables, a direct session ID, prompt identity, profile selection, and
optional appended rendered messages, then
record credential-redacted details and run that exact execution; record credential-redacted details and run that exact execution;
- return rendered debug material, validated structured output, selected - return rendered debug material, validated structured output, selected
profile, backend, effective model metadata, and token usage; profile, backend, effective model metadata, and token usage;
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
admission exhaustion through `ErrCapacityExceeded`. admission exhaustion through `ErrCapacityExceeded`.
The pinned The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/backends.go) [`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0/backends.go)
owns the registration and backend-capacity contract. owns the registration and backend-capacity contract.
For one completion, the adapter calls `PrepareExecution`, takes a For one completion, the adapter calls `PrepareExecution`, takes a
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
field, which is authoritative for provider session behavior. It also retains field, which is authoritative for provider session behavior. It also retains
the same value as the `session_id` prompt variable for maintained prompt the same value as the `session_id` prompt variable for maintained prompt
compatibility. The generated identifier is 76 ASCII characters, within compatibility. The generated identifier is 76 ASCII characters, within
PromptKit v0.8.0's 256-code-point session limit. Session IDs are non-secret PromptKit's 256-code-point session limit. Session IDs are non-secret
correlation identifiers and may be exposed to providers and provider correlation identifiers and may be exposed to providers and provider
observability. The CLI contract owns generation and override behavior. observability. The CLI contract owns generation and override behavior.
@@ -85,12 +86,54 @@ configuration and deployment workflow are defined in
[Operations](../operations.md#promptkit-profile-deployment). [Operations](../operations.md#promptkit-profile-deployment).
PromptKit owns `base_profile` resolution under its PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md). [pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403. reach the provider without authorization, which can result in a 401 or 403.
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source, PromptKit accepts only the `developer`, `system`, `user`, and
`assistant` text-chat roles after normalizing case and surrounding whitespace.
Maintained Notarius prompt definitions use only `system` and `user`.
For application-owned semantic correction, Notarius uses PromptKit's
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
exactly two messages in order: the latest validated producer response with
role `assistant`, then deterministic validation guidance with role `user`.
It never exposes a general caller-selected role API, accumulates earlier
correction turns, or changes the ordinary prompt prefix. Ordinary requests
leave appended messages unset.
PromptKit preserves supplied content but does not own Notarius's correction
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
1,114,112 bytes) before preparing the request. The transport-neutral
application contract owns defensive copying and these limits. Default request
and terminal summaries retain only safe counts, digests, identities, and usage;
complete appended messages remain limited to the explicitly requested detailed
debug trace.
PromptKit obtains its maintained OpenRouter and Rakestrawhome backend and
profile catalogs from independently versioned transitive modules. Notarius
does not import or register either catalog; PromptKit retains catalog source,
identity, precedence, credential, and capacity ownership.
External catalog profile names and settings can change between dependency
releases, including removal without aliases. Review directly selected IDs and
inherited `base_profile` values when upgrading. An application-owned alias that
inherits an external profile does not freeze that profile's settings.
PromptKit's opt-in `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` names provide name continuity within its
selected-and-tested dependency scope, not a fixed model or provider availability
guarantee. See the upstream
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.10.0/docs/release.md#catalog-profile-compatibility).
Use a complete operator-owned profile with explicit backend or endpoint, model,
and settings when those choices must remain operator-controlled. The embedded
`dnd-extraction` fallback already follows this model and does not inherit an
external catalog profile. An upgrade updates Notarius's compiled-in catalog
checkpoint identity so prior profile-source checkpoints are not reused.
Notarius supports this boundary against PromptKit v0.10.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter, implicitly supported. A dependency upgrade requires reviewing the adapter,

View File

@@ -9,36 +9,58 @@ Command syntax, streams, and exit statuses are defined in the
## Schema ## Schema
The current schema version is `notarius.run-result.v1`. The current schema version is `notarius.run-result.v2`.
| Field | Required | Meaning | | Field | Required | Meaning |
| --- | --- | --- | | --- | --- | --- |
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. | | `schema_version` | Yes | Exactly `notarius.run-result.v2`. |
| `run_id` | Yes | The finalized Notarius run identifier. | | `run_id` | Yes | The finalized Notarius run identifier. |
| `pipeline_id` | Yes | The effective pipeline identifier. | | `pipeline_id` | Yes | The effective pipeline identifier. |
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. | | `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. | | `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
| `normalized_output_count` | Yes | Number of final normalized outputs. | | `normalized_output_count` | Yes | Number of final normalized outputs. |
| `rejected_output_count` | Yes | Number of recorded rejected outputs. | | `rejected_output_count` | Yes | Number of recorded rejected outputs. |
| `warning_count` | Yes | Number of final run warnings. | | `warning_group_count` | Yes | Exact number of actionable warning groups. |
| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. |
| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. |
| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. |
| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. |
| `validation_status` | Yes | The final run manifest validation status. | | `validation_status` | Yes | The final run manifest validation status. |
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. | | `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
For the production `json` output module, `index_file` is present only when the For the production `json` output module, `index_file` is present only when the
completed run returned exactly one logical output file named `index.json`. completed run returned exactly one logical output file named `index.json`.
For another output module, its absence does not indicate a failed run. For another output module, its absence does not indicate a failed run.
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
means one or more otherwise accepted results advanced under validator-failure
`warn_continue`.
```json ```json
{ {
"schema_version": "notarius.run-result.v1", "schema_version": "notarius.run-result.v2",
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef", "run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
"pipeline_id": "dnd-session", "pipeline_id": "dnd-session",
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef", "output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
"index_file": "index.json", "index_file": "index.json",
"normalized_output_count": 6, "normalized_output_count": 6,
"rejected_output_count": 2, "rejected_output_count": 2,
"warning_count": 1, "warning_group_count": 1,
"validation_status": "rejected" "warning_occurrence_count": 2,
"diagnostic_group_count": 3,
"diagnostic_occurrence_count": 5,
"diagnostics_truncated": false,
"validation_status": "incomplete",
"validation_summaries": [
{
"stage": "extract",
"lane_id": "spells",
"status": "incomplete",
"incomplete_validators": ["dnd/spells/source_refs"],
"producer_attempt_count": 1,
"terminal_action": "warn_continue"
}
]
} }
``` ```
@@ -48,8 +70,11 @@ For another output module, its absence does not indicate a failed run.
paths. They identify the paths used by Notarius and do not resolve symlinks. paths. They identify the paths used by Notarius and do not resolve symlinks.
`output_directory` is the run-specific bundle, not the configured output root. `output_directory` is the run-specific bundle, not the configured output root.
The receipt is a summary and discovery document. It does not contain lane The receipt is a summary and discovery document. Its optional validation
descriptors, payloads, manifest data, rejections, warnings, or file contents. summaries contain only stable status, identity, validator names, reason codes,
attempt counts, and terminal actions. It does not contain lane descriptors,
payloads, manifest payloads, rejection messages, warnings, raw model responses,
correction guidance, or file contents.
For the production JSON output, resolve `index_file` beneath For the production JSON output, resolve `index_file` beneath
`output_directory`, reject path escapes, and use the `output_directory`, reject path escapes, and use the
[Published JSON Output contract](json-output.md) to discover logical files and [Published JSON Output contract](json-output.md) to discover logical files and

View File

@@ -25,7 +25,8 @@ preparation, and runner mechanics after their inputs are supplied.
## Dispatch And Configuration Handoff ## Dispatch And Configuration Handoff
The root dispatcher handles help, version reporting, configuration validation, The root dispatcher handles help, version reporting, configuration validation,
pipeline listing, and a pipeline run. Version reporting resolves build effective-configuration inspection, explanation inspection, pipeline listing,
and a pipeline run. Version reporting resolves build
information through `internal/buildinfo` before production composition, so the information through `internal/buildinfo` before production composition, so the
diagnostic remains available without configuration or runtime collaborators. diagnostic remains available without configuration or runtime collaborators.
The public syntax, streams, exit classes, and version semantics are defined by The public syntax, streams, exit classes, and version semantics are defined by
@@ -33,14 +34,15 @@ the [CLI reference](../cli.md). Other root commands normalize injectable
options before dispatch so that a missing production dependency fails as a options before dispatch so that a missing production dependency fails as a
command error rather than reaching execution. command error rather than reaching execution.
Commands that need configuration use one shared loader. The CLI discovers the Commands that need configuration use one shared ConfigKit-backed loader. The
file, parses it through **internal/core/config**, starts from defaults, applies CLI discovers the file, passes the requested profile and only explicit
the file and supported environment overrides, and then validates it for the configuration-valued run overrides, and receives a **LoadedConfig**. The
command. The configured discovery and precedence contract is in configured discovery and precedence contract is in
[Configuration](../config.md), while the loading and resolution mechanics are [Configuration](../config.md), while the loading and resolution mechanics are
in [Configuration Internals](configuration.md). in [Configuration Internals](configuration.md).
Configuration validation without a selected pipeline checks structural `config effective` and `config explain` only render the loaded ConfigKit view;
they do not resolve pipelines or construct providers. Configuration validation without a selected pipeline checks structural
configuration only. Validation with a selected pipeline also builds the configuration only. Validation with a selected pipeline also builds the
effective catalog, resolves the pipeline, and verifies every explicit effective effective catalog, resolves the pipeline, and verifies every explicit effective
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
@@ -84,8 +86,7 @@ field.
After parsing and validating a run invocation, the CLI performs this ordered After parsing and validating a run invocation, the CLI performs this ordered
handoff: handoff:
1. load and validate configuration, then apply command-level operational 1. load and validate configuration with ConfigKit command overrides;
overrides;
2. create and validate a safe run identity, then allocate a debug bundle only 2. create and validate a safe run identity, then allocate a debug bundle only
when requested; when requested;
3. build the effective catalog, resolve requested reference changes, resolve 3. build the effective catalog, resolve requested reference changes, resolve
@@ -115,6 +116,16 @@ to checkpoint identity and `pipeline.RunInput`. The public flag and stability
contract are defined by the [CLI reference](../cli.md#run); framework and LLM contract are defined by the [CLI reference](../cli.md#run); framework and LLM
packages only transport the supplied value. packages only transport the supplied value.
The CLI also owns the scope grammar for reference flags. It enumerates the
selected chunk and lane targets from registered module specifications, expands
pipeline- and lane-scoped actions into exact stage-and-lane bindings, and
resolves overlapping bind and unbind actions by specificity before calling
configuration resolution. The generic pipeline therefore receives only exact
`ReferenceBinding` and `ReferenceUnbind` values and has no knowledge of CLI
selector syntax. Configuration resolution retains ownership of configured
external/generated conflicts, required slots, and module compatibility; file
materialization still occurs afterward.
For `run --json`, the CLI constructs and encodes its private run-result receipt For `run --json`, the CLI constructs and encodes its private run-result receipt
after a successful runner result is available, before it publishes logical after a successful runner result is available, before it publishes logical
output files. It writes the prepared receipt to standard output only after output files. It writes the prepared receipt to standard output only after
@@ -166,8 +177,9 @@ is discoverable.
- **internal/cli/production_contract_test.go** covers registrar composition, - **internal/cli/production_contract_test.go** covers registrar composition,
production catalog contents, assets, and representative configuration production catalog contents, assets, and representative configuration
validation. validation.
- **internal/cli/reference_contract_test.go** covers CLI reference overrides, - **internal/cli/reference_contract_test.go** covers scoped CLI reference
origin separation, and materialization boundaries. expansion, specificity, bind/unbind conflicts, generated-reference
protection, origin separation, and materialization boundaries.
- **internal/cli/state_hardening_test.go** covers safe run identity, state - **internal/cli/state_hardening_test.go** covers safe run identity, state
roots, and failure ordering. roots, and failure ordering.

View File

@@ -8,17 +8,17 @@ either interface.
## Boundary ## Boundary
The configuration package turns a selected YAML file and supported environment The configuration package turns a selected ConfigKit-composed YAML document
values into a validated, independently owned configuration. It then resolves a into a validated, independently owned configuration. It then resolves a
requested pipeline against a module catalog before the framework prepares or requested pipeline against a module catalog before the framework prepares or
runs anything. runs anything.
| Boundary | Inputs | Outputs | Does not own | | Boundary | Inputs | Outputs | Does not own |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| Loading | Selected file path and environment lookup | Parsed file model and a populated **Config** | Choosing the file path or reporting a command result. | | Loading | Selected file path, profile, overrides, and invocation directory | **LoadedConfig** with ConfigKit inspection metadata and a populated **Config** | Choosing the file path or reporting a command result. |
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. | | Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. | | Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
| Summary | **Config** or **EffectiveConfig** | Detached redacted payload suitable for debug summaries | Redacting arbitrary process state or provider traffic. | | Inspection | **LoadedConfig** and an optional JSON Pointer | Redacted effective YAML or explanation text | Pipeline selection, module preparation, or provider work. |
The CLI discovers a configuration file, invokes this package, and supplies the The CLI discovers a configuration file, invokes this package, and supplies the
result to the framework. Configuration never reads an input file, constructs a result to the framework. Configuration never reads an input file, constructs a
@@ -28,23 +28,15 @@ at their respective [CLI](cli.md), [pipeline](pipeline.md), and
## Loading And Validation ## Loading And Validation
The CLI loads configuration in this order: ConfigKit acquires the selected root, imports, optional selected profile, and
explicit CLI overrides in precedence order. Its document definition supplies
1. parse the selected YAML file strictly into the file model; typed defaults, source-aware path normalization, and structural validation
2. start from **Default**; before conversion to the independently owned runtime **Config**. Only
3. apply the file model; and `NOTARIUS_CONFIG` participates in discovery; retired field-level environment
4. apply the supported environment overrides. variables do not participate in loading. Version 5 decoding rejects scalar
module bindings and scalar external references. ConfigKit retains redacted,
This establishes the public precedence order without giving environment input a historical inspection metadata while **LoadedConfig** keeps that implementation
second file schema. Loading and application reject malformed YAML, unsupported detail private behind effective and explanation writers.
file versions, unknown fields, invalid values, and identifiers that are empty
or collide after whitespace normalization. The file application also makes the
effective extraction-worker default follow the effective LLM limit. A present
PromptKit local-backend object requires and trims its endpoint, defaults its
omitted concurrency limit to zero, and is copied so the parsed file model
cannot alias the populated **Config**. A pipeline `llm_profile` is
presence-aware: omission remains empty, while a present blank value is
rejected and a non-empty file value is trimmed before it reaches **Config**.
**Config.Validate** checks configuration-only invariants before resolution. It **Config.Validate** checks configuration-only invariants before resolution. It
rejects incompatible profile sources, invalid state-surface values, unsupported rejects incompatible profile sources, invalid state-surface values, unsupported
@@ -57,13 +49,14 @@ It deliberately does not require modules to be registered: that requires a
catalog and belongs to resolution. catalog and belongs to resolution.
The exact user-selectable values and validation rules are defined in The exact user-selectable values and validation rules are defined in
[Configuration](../config.md). Keep additions to the file model, an [Configuration](../config.md). Keep document schema, source-aware normalization,
environment override, its validation, and that reference in the same change. runtime conversion, and that reference aligned in the same change.
## Effective Resolution ## Effective Resolution
**Config.Resolve** first recomputes derived concurrency defaults and validates **Config.Resolve** derives an absent extraction-worker value in an owned copy
the configuration. It normalizes the requested pipeline ID, copies the selected and validates the configuration. A programmatically supplied worker value is
always explicit. It normalizes the requested pipeline ID, copies the selected
profile, and passes the non-empty command-level LLM profile override, requested profile, and passes the non-empty command-level LLM profile override, requested
lane selection, and reference changes to the framework resolver. lane selection, and reference changes to the framework resolver.
@@ -74,10 +67,19 @@ profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by public field definitions and precedence are owned by
[Configuration](../config.md#pipelines). [Configuration](../config.md#pipelines).
The resolver retains configured `validation_policy` overrides and derives one
detached concrete terminal policy for the chunk producer and every lane's
extract, merge, and normalize producers. That field-by-field inheritance is
complete before preparation, and the effective values contribute to pipeline
and checkpoint identity; execution does not interpret configuration defaults.
The framework resolver supplies defaults, selects lanes, resolves validator The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting options, and returns the fixed ordered pipeline shape. Positive validator retry
**EffectiveConfig** retains the selected ID, requested selection and reference budgets require an LLM-backed selected validator; deterministic validators are
rejected during resolution. Eligible LLM-backed producer specifications also
contribute their declared correction protocol to the resolved metadata. The
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
changes, a clone of the input configuration, and the resolved pipeline. changes, a clone of the input configuration, and the resolved pipeline.
Callers may therefore retain or modify their input slices and maps without Callers may therefore retain or modify their input slices and maps without
changing the resolved result, and later consumers cannot mutate the original changing the resolved result, and later consumers cannot mutate the original
@@ -94,50 +96,44 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
The framework assigns the resolved pipeline a deterministic SHA-256 digest The framework assigns the resolved pipeline a deterministic SHA-256 digest
after defaults, lane selection, module bindings, reference bindings, validator after defaults, lane selection, module bindings, reference bindings, validator
chains, effective LLM profiles, and artifact schema identity have been chains, selected correction protocols, effective LLM profiles, and artifact
resolved. The digest excludes schema identity have been resolved. The digest excludes
its own stored value. It identifies resolved composition rather than raw YAML its own stored value. It identifies resolved composition rather than raw YAML
bytes, a debug payload, or all runtime state. The CLI records it as invocation bytes, a debug payload, or all runtime state. The CLI records it as invocation
provenance before execution; cache and checkpoint identity have additional provenance before execution; cache and checkpoint identity have additional
owners in [Run State Internals](state.md). owners in [Run State Internals](state.md).
Configuration summaries must use **Redacted**, **RedactedSummaryPayload**, or Configuration publication uses ConfigKit's redacted effective YAML. Resolved
**RedactedResolvedPipelinePayload**, never a direct configuration marshal. pipeline publication uses **RedactedResolvedPipelinePayload**, which copies
Those methods copy every binding and nested option container, replace values bindings and nested option containers, replaces credential-shaped values with
whose key is credential-shaped with **[REDACTED]**, and omit materialized **[REDACTED]**, and omits materialized reference content while retaining safe
reference content while retaining safe binding and reference provenance. The provenance. This redaction is deliberately narrow and does not authorize
payload must not alias the source configuration or resolved pipeline. recording arbitrary environment values or provider requests.
PromptKit's local endpoint and concurrency limit are preserved as non-secret
configuration metadata in the independently owned summary; the object contains
no credential value. This redaction is deliberately narrow: it protects
configuration summaries and does not authorize recording arbitrary environment
values or provider requests.
## Invariants To Preserve ## Invariants To Preserve
- Defaults, YAML values, and environment values are applied in one direction; - ConfigKit composition applies defaults, imports, an optional profile, and
later sources may override only their supported operational settings. explicit overrides in one direction with retained origin metadata.
- A configuration is structurally valid before it is resolved, and a resolved - A configuration is structurally valid before it is resolved, and a resolved
pipeline is compatible with the supplied catalog before preparation begins. pipeline is compatible with the supplied catalog before preparation begins.
- Whitespace-normalized identifiers are unique wherever they identify a - Whitespace-normalized identifiers are unique wherever they identify a
pipeline, step, lane, worker, or reference slot. pipeline, step, lane, worker, or reference slot.
- Resolution and summary generation return detached data. Redaction must cover - Runtime conversion and resolved-pipeline publication return detached data.
every configured and resolved binding, including nested validator bindings.
- The resolved digest changes when resolved composition changes and never - The resolved digest changes when resolved composition changes and never
includes itself. includes itself.
## Focused Tests ## Focused Tests
- **internal/core/config/file_config_contract_test.go** covers strict file - **internal/core/config/document_test.go** and
parsing, normalization, file application, and structural rejection. **operational_document_test.go** cover ConfigKit document loading,
- **internal/core/config/env_contract_test.go** covers supported operational normalization, imports, profiles, and overrides.
overrides and their precedence.
- **internal/core/config/validation_contract_test.go** covers configuration - **internal/core/config/validation_contract_test.go** covers configuration
invariants and contextual failures. invariants and contextual failures.
- **internal/core/config/effective_config_contract_test.go** covers defaults, - **internal/core/config/effective_config_contract_test.go** covers defaults,
selections, overrides, resolution context, digest changes, and ownership. selections, overrides, resolution context, digest changes, and ownership.
- **internal/core/config/redaction_test.go** covers recursive credential - **internal/core/config/loaded_config_test.go** covers safe effective and
redaction, reference-content exclusion, and non-aliasing payloads. explanation rendering; **redaction_test.go** covers resolved-pipeline
redaction and reference-content exclusion.
Run **go test ./internal/core/config** after changing this boundary. Changes to Run **go test ./internal/core/config** after changing this boundary. Changes to
the handoff or resolved-composition semantics also need the focused framework the handoff or resolved-composition semantics also need the focused framework

View File

@@ -29,11 +29,17 @@ The D&D registrar registers the family’s artifact codecs, extractors, typed
append-order mergers, normalizers, validators, prompt assets, fallback LLM append-order mergers, normalizers, validators, prompt assets, fallback LLM
profile asset, and default validator chains. Each extractor and normalizer has profile asset, and default validator chains. Each extractor and normalizer has
a stable module spec, explicit execution class, strict option decoding, and a a stable module spec, explicit execution class, strict option decoding, and a
typed builder. Scene chunking, every extractor, and NPC, location, and item-registry typed builder. Scene chunking, every extractor, and NPC, location, and
normalization are registered as `llm_backed`; the remaining current D&D mergers item-registry normalization are registered as `llm_backed`; the remaining
and normalizers are `deterministic`. The metadata is available to catalog inspection and current D&D mergers and normalizers are `deterministic`. The metadata is
resolved-pipeline debug data and determines which selected bindings inherit the available to catalog inspection and resolved-pipeline debug data and determines
pipeline profile. Configuration remains the canonical owner of the exact keys, which selected bindings inherit the pipeline profile. The registry normalizers
use `single_response_v1`, forwarding corrections to their reconciliation
completion and retaining the accepted raw proposal only as an owned model
candidate. When deterministic proposal safety or typed application rejects a
group, they provide separate model-facing prose for a corrective module retry;
internal issue categories, reason codes, and operator messages remain
diagnostic-only. Configuration remains the canonical owner of the exact keys,
profile precedence, and validator order. profile precedence, and validator order.
Private structured-LLM response schemas are deliberately minimal. They reject Private structured-LLM response schemas are deliberately minimal. They reject
@@ -42,6 +48,13 @@ unknown fields, while preserving semantic candidates for deterministic
validation. Do not promote a private response envelope into a durable schema; validation. Do not promote a private response envelope into a durable schema;
the contracts above define durable data. the contracts above define durable data.
A D&D producer that declares `single_response_v1` forwards any supplied
semantic correction to its structured completion and returns an owned copy of
that completion's exact validated raw response as its model candidate. It does
not serialize normalized artifacts to create that candidate, so deterministic
identity, evidence, warning, and durable-schema behavior remains separate from
the model transport material.
## Prompt Construction ## Prompt Construction
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
@@ -108,14 +121,27 @@ presentation, and final ephemeral generic transcript windows. These orders and
cache controls are prompt behavior; change them only through the owning cache controls are prompt behavior; change them only through the owning
manifest and prompt declaration. manifest and prompt declaration.
NPC, item, and location registry reconciliation translate shared proposal
safety categories into bounded prose that references only the response-local
duplicate-group ordinals and candidate handles. Item reconciliation appends its
typed rule that currency may be consolidated only with aliases of the same
denomination and never with non-currency items. The next normalize attempt
receives that prose with the exact defective proposal under the shared
replacement-request protocol. Structurally invalid output has no valid proposal
candidate and receives a fresh feedback-free attempt instead. If the stage
budget is exhausted, safe groups stay applied, unsafe groups stay separate, and
one fallback warning summarizes the final defect without raw model content.
## Evidence, Candidates, And Normalization ## Evidence, Candidates, And Normalization
The current transcript is the only durable evidence source. Extractors assign The current transcript is the only durable evidence source. Extractors assign
the current source identity, preserve candidate evidence ranges for validators, the current source identity and losslessly order any reversed range whose two
and canonically order or remove exact duplicate ranges without asking the endpoints resolve in that source, using transcript position rather than numeric
model to repair semantic errors. Campaign context and generated artifacts may unit-ID order. They then canonically order ranges and remove exact duplicates.
ground names or control routing, but they never establish evidence for a D&D This routine canonicalization does not request a retry or emit a warning.
result. Unresolvable or wrong-source ranges remain unchanged for validators to reject.
Campaign context and generated artifacts may ground names or control routing,
but they never establish evidence for a D&D result.
Default chains keep responsibilities separate: structural validators assess the Default chains keep responsibilities separate: structural validators assess the
candidate, source-reference validators resolve cited ranges against the current candidate, source-reference validators resolve cited ranges against the current
@@ -125,15 +151,119 @@ relatedness validators report advisory evidence concerns. The configured order
is documented in is documented in
[Configuration](../config.md#production-validator-keys-and-default-chains). [Configuration](../config.md#production-validator-keys-and-default-chains).
The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the
D&D family's LLM-backed review of only combat versus non-combat classification.
It selects the shared combat-policy prompt fragment and asks the model to
classify the current chunk independently as `combat` or `non_combat` without
receiving the proposed scene kind. Deterministic code compares that
classification with the proposed kind and either approves it or produces the
appropriate correction guidance. This keeps every schema-valid classification
interpretable and avoids anchoring the reviewer on the producer's answer. It
does not assess titles, summaries, non-combat subtype, or scene boundaries;
deferred boundary-coherence review remains separate. It is opt-in;
[Configuration](../config.md) owns selection and retry/failure behavior.
### Combat-semantics provider evaluation
The human-reviewed corpus at
`internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json`
owns the proposed kind, expected combat classification, and reviewer rationale
for each synthetic case. Its package test validates the fixture contract only.
Provider evaluation remains an explicit maintainer operation and must not be
added to the default offline test suite.
Use the following protocol before proposing default-chain inclusion:
1. Record the Notarius commit, prompt and schema fingerprints, provider, model,
profile settings, reasoning effort, structural-repair setting, number of
repetitions, and evaluation date before collecting results. Do not revise
expected classifications merely to agree with provider output; a substantive
corpus correction requires independent human review.
2. Exercise the production validator construction and prompt assets from an
explicitly invoked, disposable evaluation driver or test in the validator
package. For each corpus case, construct transcript source units from the
listed IDs and text, assign matching per-unit source references, and use
`source.MaterializeChunkPlan` with one range spanning those units. Construct
exactly one scene whose ID and source range match that chunk and whose kind
is the case's `proposed_kind`; title and summary may use fixed placeholders
because the validator neither receives nor evaluates them. Invoke the typed
validator at extract stage through the production registry and scheduled LLM
client. Do not commit provider credentials, generated source material, or an
always-on live test.
3. Compare the model classification with `expected_classification`, then verify
that its deterministic comparison with `proposed_kind` yields the expected
approval or rejection direction.
Record an unexpected approval of an expected rejection as a false
acceptance, an unexpected rejection of an expected approval as a false
rejection, and any validator execution failure separately from semantic
accuracy. Retain per-case results so repeated trials and systematic failure
modes remain visible.
4. Evaluate producer correction separately with representative complete
scene-description runs configured as shown in
[Configuration](../config.md#production-validator-keys-and-default-chains).
For every initial semantic rejection, record whether the next producer
attempt returns the requested combat status and is approved. Do not count a
PromptKit structural repair as a producer-correction attempt.
5. Run the correction evaluation with debug capture enabled and without reused
extraction checkpoints. Record added validator and producer calls, elapsed
latency, and cumulative prompt, completion, cached, and total token usage
from the debug attempt records. Compare these values with an otherwise
identical run whose scene-description chain omits the semantic validator.
The default-chain review must consider classification error, false acceptance,
false rejection, execution failure, producer-correction success, added calls,
latency, and token use together. A structurally successful provider run alone
is not evidence that the validator should become a default.
Every producer-correctable D&D rejection describes all currently detectable
corrections in transcript-grounded domain terms, using contextual names,
model-owned artifact fields, and source segment ranges when useful. Validators
collect independent record defects in one pass so one retry does not merely
reveal the next issue. Shared D&D diagnostic helpers keep repeated rules and
record descriptions stable, de-duplicated, and bounded; each artifact family
continues to own the semantic rule and its prose.
Operator diagnostics and model guidance are separate products of the same
assessment. Operator messages may use typed paths, reason details, and opaque
application identities. Correction guidance must not copy those messages or
ask the model to reproduce durable entity IDs, hashes, validator module keys,
reason codes, or Go field paths. A registry-normalization rejection instead
speaks in terms of the duplicate-group proposal response the normalizer can
actually revise. Normalization-only deterministic invariants retain useful
operator detail but do not imply that a model controls derived ordering or
identity. Only bounded actionable semantic guidance is eligible for a
correction prompt.
Private LLM schemas use simple enums for closed categorical fields when the
provider-compatible shape can express the rule directly. Deterministic typed
validators retain the same checks as defense in depth and for non-LLM
producers. Private schemas keep every property required and avoid optional
properties, `uniqueItems`, and conditional cross-field logic.
Item-occurrence shape validation groups repeated holder mistakes by occurrence
kind and gives the producer the required JSON null/non-null relationship. It
identifies affected records by contextual item name and cited transcript range,
never by the deterministically attached durable item ID. Holder mistakes remain
semantic rejections rather than silent rewrites because changing a holder can
also change the meaning of the occurrence kind.
Enemy-event extraction additionally rejects a second `engaged` observation for Enemy-event extraction additionally rejects a second `engaged` observation for
the same comparison identity within one scene-scoped result. Normalization may the same comparison identity within one scene-scoped result. Normalization may
combine results from distinct scenes, so it intentionally does not apply that combine results from distinct scenes, so it intentionally does not apply that
rule. Configuration owns the exact validator key and chain position. rule. Configuration owns the exact validator key and chain position.
Extraction source-reference validators share one full-span chunk-containment
policy. After ordinary reference validity succeeds, the policy resolves both
endpoints through document order and requires every source unit in the
inclusive range to be present in the current chunk. It does not assume numeric
unit-ID ordering, mutate input, or weaken wrong-source and unresolved-reference
validation. Scene descriptions remain separate because their validator owns an
exact one-scene range contract rather than general extraction containment.
Normalizers are deterministic for spells, combat turns, item occurrences, NPC Normalizers are deterministic for spells, combat turns, item occurrences, NPC
occurrences, scene descriptions, enemy events, and location occurrences. They occurrences, scene descriptions, enemy events, and location occurrences. They
canonicalize display values and evidence, use source-document order for stable canonicalize display values and evidence, use source-document order for stable
output, and issue bounded warnings for changes or collapsed duplicates. NPC, output, and emit bounded normalization observations for changes or collapsed duplicates. NPC,
item, and location registry normalizers are intentional exceptions: each first item, and location registry normalizers are intentional exceptions: each first
produces a deterministic candidate set, then may use a bounded structured-LLM produces a deterministic candidate set, then may use a bounded structured-LLM
proposal to reconcile identity groups. proposal to reconcile identity groups.
@@ -142,13 +272,16 @@ proposal to reconcile identity groups.
The three registry normalizers instantiate the domain-neutral The three registry normalizers instantiate the domain-neutral
`internal/framework/semanticreconcile` engine with default bounds. Each `internal/framework/semanticreconcile` engine with default bounds. Each
eligible candidate receives a contiguous, one-based `candidate_id` for that eligible candidate receives a contiguous, one-based `candidate_number` for that
request. The model sees that handle, the candidate label and source-free request, and candidate material declares the exact inclusive range. The model
evidence ranges, plus bounded transcript windows; it returns only duplicate sees that handle, the candidate label and source-free evidence ranges, plus
bounded transcript windows; it returns only duplicate
groups of supplied handles and one supplied canonical handle per group. It groups of supplied handles and one supplied canonical handle per group. It
never returns names, evidence, durable IDs, or replacement records. Identical never returns names, evidence, durable IDs, or replacement records. Identical
labels and evidence remain independently selectable because their handles are labels and evidence remain independently selectable because their handles are
distinct. distinct. Transcript unit `id` values and evidence `start_unit_id` and
`end_unit_id` values are source positions in a separate namespace and are
never valid candidate numbers.
The generic core owns the mandatory handle protocol, candidate and transcript The generic core owns the mandatory handle protocol, candidate and transcript
presentation, the private response schema, source-reference validation, presentation, the private response schema, source-reference validation,

View File

@@ -42,6 +42,22 @@ observability. The adapter returns PromptKit’s validated raw bytes rather than
re-encoding the decoded target. An empty optional material is represented as re-encoding the decoded target. An empty optional material is represented as
one space so its named input is retained by PromptKit. one space so its named input is retained by PromptKit.
When a request includes semantic correction material, the adapter validates and
defensively copies it before preparation, then appends exactly two messages
after the ordinarily rendered prompt: the prior response as an assistant
message and the correction guidance as a user message. Requests without a
correction do not add messages or introduce caller roles. Ordinary request
summaries record correction byte counts and digests only; complete messages are
available solely in an explicitly requested debug trace.
The adapter leaves the ordinary rendered message prefix, named inputs,
variables, session, profile, execution overrides, prepared-execution path, and
PromptKit repair policy unchanged for a corrected request. It never imports a
PromptKit message type into a module or pipeline contract. PromptKit reports
actual structural repair count and cumulative token usage per completion; the
pipeline's safe terminal debug record projects those values without copying
message content.
Client construction may also receive a run-wide reasoning-effort override from Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer creates a fresh PromptKit execution override for each request: a nil pointer
@@ -229,10 +245,15 @@ or corrective call remain provider-neutral operational errors with the same
redaction boundary. redaction boundary.
Structural repair does not replace pipeline retry behavior: a binding's Structural repair does not replace pipeline retry behavior: a binding's
configured retry count reruns its complete stage attempt after an error or configured retry count reruns its complete stage attempt after an operational
rejection. The pipeline owns attempt lifecycle, validation chains, and retry or structural error, module-requested retry, or actionable semantic rejection.
diagnostics; see [Pipeline Internals](pipeline.md#validation-retries-and-output) An actionable module-requested retry and a validator rejection both use the
and the [binding reference](../config.md#module-bindings-and-validators). same correction payload when the producer exposes an exact latest response;
feedback-free module retries reconstruct the ordinary request without appended
messages. The pipeline owns attempt lifecycle, validation chains, and retry
diagnostics; see
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
[binding reference](../config.md#module-bindings-and-validators).
## Timeout Ownership ## Timeout Ownership
@@ -262,6 +283,13 @@ surfaced; when the completion already failed, its call error remains the
result. Debug-bundle location, retention, and handling are operational concerns result. Debug-bundle location, retention, and handling are operational concerns
documented in [Operations](../operations.md#debug-bundles). documented in [Operations](../operations.md#debug-bundles).
The attempt-terminal summary is a separate safe trace record: it contains
attempt kinds, validator outcome counts and reason codes, effective policy,
terminal action, and repair/usage references. It excludes raw assistant
responses and correction text. Those values can appear only in the explicitly
requested detailed prompt and response artifacts, which require sensitive-data
handling.
Run manifests receive selected profile summaries, including optional effective Run manifests receive selected profile summaries, including optional effective
backend and reasoning provenance, and component identities—not prompt, schema, backend and reasoning provenance, and component identities—not prompt, schema,
source, reference, or response content. The published field semantics belong source, reference, or response content. The published field semantics belong

View File

@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
binding-specific profile on a deterministic module. The user-facing precedence binding-specific profile on a deterministic module. The user-facing precedence
contract belongs in [Configuration](../config.md#pipelines). contract belongs in [Configuration](../config.md#pipelines).
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
declare correction protocol `single_response_v1`. That declaration is a
promise that the implementation accepts one attempt-local semantic correction
and returns an owned copy of the exact one model response that directly
controlled the candidate. It must forward correction only to its structured
completion request; it must not manufacture prior-response material by
serializing a normalized artifact or expose opaque application IDs. Input,
output, validator, and deterministic specs cannot declare the protocol.
Implementations that accept options must provide both an option validator and Implementations that accept options must provide both an option validator and
a builder. The validator is used while resolving configuration; the builder a builder. The validator is used while resolving configuration; the builder
decodes the same options and constructs the implementation from the prepared decodes the same options and constructs the implementation from the prepared
@@ -36,6 +45,15 @@ they need, register each leaf implementation, and add any family-owned assets
or default validator chains. They return contextual errors so production or default validator chains. They return contextual errors so production
composition fails at startup rather than at the first run. composition fails at startup rather than at the first run.
A validator that returns a completed rejection must supply two separate
bounded values: a stable `ReasonCode` for provenance and actionable
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
and the constraints on one complete corrected replacement. It must not contain
validator keys, diagnostic paths, opaque application IDs, or other internal
identifiers. An operator-facing `Message` may explain the same event, but the
framework never copies it into a model request. Missing or invalid guidance is
a validator contract failure.
An artifact family can register an optional typed evidence projector alongside An artifact family can register an optional typed evidence projector alongside
its codec. The projector returns defensive copies of the artifact's direct its codec. The projector returns defensive copies of the artifact's direct
generic source references and must use the codec's exact Go type. It does not generic source references and must use the codec's exact Go type. It does not
@@ -77,19 +95,40 @@ source-backed artifact-family normalizer projects its deterministic records
into contextual candidates and owned typed record envelopes, supplies its into contextual candidates and owned typed record envelopes, supplies its
chosen prompt identity and resolved LLM profile, and constructs an engine with chosen prompt identity and resolved LLM profile, and constructs an engine with
explicit limits. The core filters invalid evidence, assigns contiguous explicit limits. The core filters invalid evidence, assigns contiguous
request-local integer handles, renders bounded candidate and transcript one-based request-local candidate numbers, renders bounded candidate and
materials, invokes the structured-completion boundary, and assesses the transcript materials, invokes the structured-completion boundary, and assesses
returned duplicate groups into a stable non-overlapping plan. the returned duplicate groups into a stable non-overlapping plan. Candidate
material declares the exact inclusive number range for the request. Transcript
unit IDs and evidence range endpoints remain source positions in a separate
namespace and are never valid candidate numbers.
The normalizer then applies that plan through a typed `ApplicationPolicy`. The The normalizer then applies that plan through a typed `ApplicationPolicy`. The
core preserves ungrouped records, contribution order, and provenance while the core preserves ungrouped records, contribution order, and provenance while the
artifact family owns group guards, field and evidence consolidation, durable artifact family owns group guards, field and evidence consolidation, durable
ID derivation, retry and fallback presentation, warnings, and postconditions. ID derivation, retry and fallback presentation, classified diagnostics, and
Request-local handles do not enter the typed value or durable artifact. Fewer postconditions.
than two eligible candidates skips model invocation; exceeding a candidate or These candidate numbers are the concrete private representation of ADR-0013's
combined-material bound preserves the deterministic result under the family's request-local handles; they do not enter the typed value or durable artifact.
fallback policy. Provider, transport, cancellation, and context-construction Fewer than two eligible candidates skips model invocation; exceeding a
failures remain execution errors. candidate or combined-material bound preserves the deterministic result under
the family's fallback policy. Provider, transport, cancellation, and
context-construction failures remain execution errors.
When the engine actually makes a proposal call, its typed result carries the
owned exact proposal response under the same correction contract as other
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
model candidate, so a later rejection applies terminal policy without spending
an ineffective semantic retry.
When proposal assessment or typed group application rejects a structurally
valid group, the normalizer may return its safe partial value with a
module-requested retry. A feedback-capable directive supplies bounded
model-facing correction guidance separately from operator diagnostics and
retains the exact proposal response as its candidate. The shared stage retry
mechanism appends that response and guidance to a fresh complete request. A
structurally invalid completion has no valid candidate and therefore requests a
feedback-free fresh attempt. On exhaustion, only the final safe fallback and
its bounded process diagnostic advance to validation.
The core supplies a conservative generic prompt and the single private The core supplies a conservative generic prompt and the single private
response schema. A domain prompt may substitute its semantic instructions but response schema. A domain prompt may substitute its semantic instructions but
@@ -111,9 +150,13 @@ its domain prompt.
3. Implement strict option decoding, construction, and the typed stage 3. Implement strict option decoding, construction, and the typed stage
interface. Preserve caller ownership: do not retain mutable request data interface. Preserve caller ownership: do not retain mutable request data
and return defensive copies where an implementation exposes stored data. and return defensive copies where an implementation exposes stored data.
If declaring correction capability, forward the request correction and
retain only the exact validated response that controlled the result.
4. Register the module through its typed registry helper and add it to the 4. Register the module through its typed registry helper and add it to the
owning family registrar. Add a default validator chain only when that owning family registrar. Add a default validator chain only when that
family owns the behavior; otherwise require an explicit compatible chain. family owns the behavior; otherwise require an explicit compatible chain.
Every rejection path in a validator must provide actionable correction
guidance while retaining its stable internal reason code.
5. Update the selectable-key and chain reference in 5. Update the selectable-key and chain reference in
[Configuration](../config.md#production-module-keys), the applicable [Configuration](../config.md#production-module-keys), the applicable
integration contract, and focused tests. Keep the configuration document integration contract, and focused tests. Keep the configuration document

View File

@@ -26,7 +26,7 @@ physical state roots.
| --- | --- | --- | | --- | --- | --- |
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. | | Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
| Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. | | Build information | **internal/buildinfo** | Resolves a stable linked release tag or build metadata for the diagnostic root version command. |
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. | | Configuration | **internal/core/config** | ConfigKit document definition, source-aware normalization, runtime conversion, structural validation, safe inspection, effective resolution, and resolved-composition redaction. |
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. | | Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. | | Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. | | LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |

View File

@@ -12,7 +12,7 @@ own durable output shapes. Concrete production extensions are covered by
The pipeline framework accepts a resolved composition, registries, shared The pipeline framework accepts a resolved composition, registries, shared
dependencies, input bytes, a supplied prompt session, and state/debug dependencies, input bytes, a supplied prompt session, and state/debug
collaborators. It returns logical output files, normalized artifacts, recorded collaborators. It returns logical output files, normalized artifacts, recorded
rejections and warnings, manifest provenance, and checkpoint decisions. The rejections, grouped diagnostics, manifest provenance, and checkpoint decisions. The
CLI owns process arguments, configuration discovery, session resolution, CLI owns process arguments, configuration discovery, session resolution,
physical roots, and placement of returned output files. physical roots, and placement of returned output files.
@@ -32,8 +32,14 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
It normalizes the pipeline and lane identities, applies stage defaults, selects It normalizes the pipeline and lane identities, applies stage defaults, selects
requested lanes where that is supported, resolves validator chains, checks requested lanes where that is supported, resolves validator chains, checks
module capabilities and typed artifact compatibility, validates options, and module capabilities and typed artifact compatibility, validates options, and
assigns a deterministic resolved-composition digest. The resolved pipeline assigns a deterministic resolved-composition digest. A correction protocol is
contains bindings and declared reference targets, not external reference bytes. selected from each eligible LLM-backed producer specification and becomes part
of that resolved identity; only `single_response_v1` is currently supported.
Preparation rejects an LLM-backed producer that combines a non-empty validator
chain with positive producer retries unless it declares that protocol. Producers
without validators or without retries remain valid without correction support.
The resolved pipeline contains bindings and declared reference targets, not
external reference bytes.
After selection, the resolver applies command, binding, and pipeline profile After selection, the resolver applies command, binding, and pipeline profile
precedence to LLM-backed bindings and validators only; prompt defaults remain precedence to LLM-backed bindings and validators only; prompt defaults remain
an empty resolved binding profile. It resolves structural output repair an empty resolved binding profile. It resolves structural output repair
@@ -59,9 +65,11 @@ remains declared but has no bytes until its producing step completes.
Preparation is the construction boundary. It validates the resolved shape and Preparation is the construction boundary. It validates the resolved shape and
registry set, clones the resolved data, then constructs the input adapter, registry set, clones the resolved data, then constructs the input adapter,
chunker, stage-local validators, every typed lane, and output encoder. Each chunker, stage-local validators, every typed lane, and output encoder. The
registered builder receives its own cloned build request immediately before its prepared producer metadata preserves each selected correction protocol, and
module-owned code runs. Preparation also collects stable checkpoint the resolved digest carrying that metadata participates in checkpoint identity.
Each registered builder receives its own cloned build request immediately
before its module-owned code runs. Preparation also collects stable checkpoint
fingerprints. Missing registrations, incompatible typed entries, nil fingerprints. Missing registrations, incompatible typed entries, nil
implementations, and constructor failures are reported before source parsing implementations, and constructor failures are reported before source parsing
or any stage operation begins. or any stage operation begins.
@@ -89,6 +97,13 @@ codec, checks its complete schema and media identity, and records a content
digest plus bounded producer provenance. A missing, ambiguous, invalid, or digest plus bounded producer provenance. A missing, ambiguous, invalid, or
incompatible producer prevents the consumer step from starting. incompatible producer prevents the consumer step from starting.
Resolution receives only exact stage-and-lane operational reference overrides.
The CLI may offer broader pipeline- or lane-scoped selectors, but expands and
arbitrates those before entering the framework. External overrides are applied
after configured external defaults and local bindings. They cannot coexist
with a generated binding for the same target and slot, and external unbinds do
not remove generated handoffs.
## Execution And Ordering ## Execution And Ordering
The runner validates its input, installs no-op state collaborators when none The runner validates its input, installs no-op state collaborators when none
@@ -128,17 +143,55 @@ their target: chunks, codec-decoded typed candidates, or serialized codec
bytes. Each typed validator receives a newly decoded value from the one bytes. Each typed validator receives a newly decoded value from the one
candidate serialization for that attempt, while serialized validators receive candidate serialization for that attempt, while serialized validators receive
separately owned representation bytes and schema metadata. They may approve, separately owned representation bytes and schema metadata. They may approve,
approve with warnings, reject, or fail. A rejection is an ordinary pipeline approve with warnings, reject, fail, or be skipped when a runtime prerequisite
result; a validator error is a framework error. is unavailable. The shared executor settles every configured validator in
order. A failed LLM-backed validator retries only itself against the same
immutable candidate; it does not regenerate the producer or alter the
validator request. Rejections stop that validator, while other configured
validators still run. The executor retains ordered results, bounded
deduplicated correction guidance from rejections, and only the final exhausted
failure outcome for each validator. The correction builder keeps first
occurrence order, omits internal reason codes, validator names, and operator
messages, and requests one complete replacement. Missing guidance or an
oversized aggregate is a framework contract error; guidance is never inferred
or truncated.
The runner applies the binding's retry policy around a stage operation and its The runner applies the binding's retry policy around a stage operation and its
complete validation chain. It preserves warnings only from the final accepted complete validation chain. It preserves terminal diagnostics only from the final accepted
or rejected attempt. Cancellation stops retries. Normalizer-specific retry or rejected attempt, plus one fixed validation-incomplete warning per validator whose execution
directives consume this same budget and validate any final safe fallback through budget was exhausted under `warn_continue`. Cancellation stops retries.
the normalizer chain. Normalizer-specific retry directives consume this same budget and validate any
final safe fallback through the normalizer chain. A directive may carry bounded
correction guidance only when it also exposes the exact latest
`single_response_v1` candidate. The state machine then uses the same replacement
request shape as validator correction. A directive without guidance clears any
prior correction and starts a fresh attempt, which preserves structural retry
behavior when no valid response exists.
The artifact-neutral producer-attempt state machine owns that shared budget,
attempt provenance, semantic-correction material, and terminal-policy
selection. It accepts producer and complete-validation closures, so artifact
materialization, cache handling, checkpoints, and debug output stay at the
operation boundary. It distinguishes operational, structural, module-requested,
and validator-semantic retries. Model feedback from either semantic source is
available only for a valid latest `single_response_v1` candidate. A
deterministic or no-model validator rejection instead settles the semantic
policy immediately, while a feedback-free module directive remains an ordinary
fresh retry. Structural-output errors alone use the structural policy, and
validation failure without rejection settles the validator-failure policy
without regenerating the producer.
Chunk planning uses this state machine for generated plans. A rejected or
validation-incomplete automatic cache hit is not model material and therefore
falls through to a fresh initial generation at producer attempt one; it neither
receives a correction, consumes retry budget, promotes cached-candidate
diagnostics, nor overwrites the stored record. An incomplete cache validation
under `fail_run` terminates instead. Only a newly generated, completely
validated plan is published to the chunk-plan store. Rejected plans never
advance, and validation-incomplete plans remain unpublishable.
After terminal lane work, the runner assembles manifest provenance, normalized After terminal lane work, the runner assembles manifest provenance, normalized
artifacts, rejections, warnings, and an optional accepted chunk map. When an artifacts, rejections, final grouped diagnostics, and an optional accepted chunk map. When an
output policy selected evidence lanes, it decodes accepted serialized normalize output policy selected evidence lanes, it decodes accepted serialized normalize
outputs through their registered codecs and invokes the prepared typed outputs through their registered codecs and invokes the prepared typed
projectors. Rejected or absent lanes contribute nothing. This reconstruction is projectors. Rejected or absent lanes contribute nothing. This reconstruction is
@@ -149,6 +202,15 @@ The CLI publishes those files only after the runner returns without a framework
error. Logical file names and schemas are defined by the [output integration error. Logical file names and schemas are defined by the [output integration
contracts](../integrations/). contracts](../integrations/).
For every completed producer disposition, the runner projects one bounded
validation summary to the manifest, the affected rejection when present, and
the CLI result receipt. The summary records status, configured-order rejecting
validators and reason codes, incomplete validators, producer-attempt count,
and terminal action. It contains no operator message, correction guidance, or
model response. `complete`, `rejected`, and `incomplete` describe the final
candidate disposition; a run-level `incomplete` status indicates at least one
current-run output advanced under `warn_continue`.
## Checkpoint And Debug Hooks ## Checkpoint And Debug Hooks
The runner receives checkpoint and debug interfaces rather than roots. It The runner receives checkpoint and debug interfaces rather than roots. It
@@ -158,10 +220,26 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
Selective recomputation can require a canonical accepted normalized predecessor Selective recomputation can require a canonical accepted normalized predecessor
before a dependent lane starts. before a dependent lane starts.
The runner writes successful checkpoint artifacts only after complete accepted
validation. Chunk plans follow the same rule for publication. A rejection,
invalid structured response, or incomplete validation is never reusable state;
the current run may still hand off an otherwise valid `warn_continue` result
according to its terminal policy. The runner carries private reuse eligibility
through extract, merge, normalize, and generated-reference handoff. Any stage
derived from incomplete validation skips both checkpoint lookup and all
checkpoint publication even when that stage's own validation completes.
External references and fully validated generated references remain eligible.
Attempt debug records retain safe kind,
validator, repair-usage, policy, and terminal-decision provenance. Full
assistant and correction content remains confined to the requested detailed
LLM trace.
Debug recording is attempt-scoped and application-owned. A failure to persist Debug recording is attempt-scoped and application-owned. A failure to persist
required debug data is a framework error. State roots, persistence, reason-code required debug data is a framework error. State roots, persistence, reason-code
meanings, resume, and cleanup are intentionally owned by meanings, resume, and cleanup are intentionally owned by
[Run State Internals](state.md) and [Operations](../operations.md). [Run State Internals](state.md) and [Operations](../operations.md).
Extract-validator trace scopes include the current chunk ordinal so concurrent
chunks cannot overwrite one another's validator attempts or LLM artifacts.
## Invariants To Preserve ## Invariants To Preserve

View File

@@ -32,11 +32,18 @@ owned by the [Accepted Chunk Map contract](../integrations/chunk-map.md).
`internal/framework/chunkplan` owns source-addressed plan storage, validation, `internal/framework/chunkplan` owns source-addressed plan storage, validation,
and atomic publication. Its store is constructed only when the selected mode is and atomic publication. Its store is constructed only when the selected mode is
not `bypass`. not `bypass`. It preserves rooted entry checks while creating new cache roots
and source-digest directories with requested mode `0777`, then creates an
atomic temporary file through rooted `O_CREATE|O_EXCL` with requested mode
`0666` before sync and rename.
`internal/framework/checkpoint` owns checkpoint identity, manifests, payload `internal/framework/checkpoint` owns checkpoint identity, manifests, payload
codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint
recording is enabled and constructs a loader only for a `--resume` invocation. recording is enabled and constructs a loader only for a `--resume` invocation.
The recorder writes manifest and payload JSON through
`fileio.WriteJSONWithProcessPermissions`, whose confined atomic writer creates
new directories with requested mode `0777` and temporary files with requested
mode `0666` before publication.
Identity incorporates explicit stable semantic fingerprints collected from Identity incorporates explicit stable semantic fingerprints collected from
prepared modules and validators in addition to configuration, input, prepared modules and validators in addition to configuration, input,
references, runtime overrides, observed LLM profiles, and the LLM runtime's references, runtime overrides, observed LLM profiles, and the LLM runtime's
@@ -64,12 +71,13 @@ Ordinary resume loads extract, merge, and normalize checkpoints progressively
and may execute later lane stages after an earlier cache miss. Selective and may execute later lane stages after an earlier cache miss. Selective
recomputation instead asks the loader for the required producer's accepted recomputation instead asks the loader for the required producer's accepted
normalize artifact. That lookup reuses the existing normalize files, requires normalize artifact. That lookup reuses the existing normalize files, requires
workspace schema v3 plus an exact non-empty invocation identity, and deliberately workspace schema v4 plus an exact non-empty invocation identity, and deliberately
does not require extract or merge checkpoint files or dependency fingerprints. does not require extract or merge checkpoint files or dependency fingerprints.
The runner performs canonical codec and producer-provenance validation before The runner performs canonical codec and producer-provenance validation before
cloning the artifact into normal step output. Success restores only stored cloning the artifact into normal step output. Success restores only stored
normalize warnings and emits one normalize decision; failure retains the files, normalize diagnostics and emits one normalize decision; failure retains the
records the decision, and stops without executing the producer or consumer. files, records the decision, and stops without executing the producer or
consumer.
The loader assigns a typed category and reason code at each validation site; The loader assigns a typed category and reason code at each validation site;
diagnostic prose is not classified after the fact. The runner then applies diagnostic prose is not classified after the fact. The runner then applies
@@ -85,7 +93,10 @@ and debug-summary JSON.
canonical operator-facing reason-code reference. canonical operator-facing reason-code reference.
`internal/core/fileio` provides confined atomic file writes used by state `internal/core/fileio` provides confined atomic file writes used by state
collaborators. The chunk-plan store retains its stronger entry validation. collaborators. Its exact-mode writer remains available to output and debug
callers; its process-permission writer is used by checkpoints. The chunk-plan
store retains its stronger rooted entry validation. [Operations](../operations.md#cache-provisioning)
owns cache mode and provisioning guidance.
The CLI constructs selective-recomputation policy from resolved generated The CLI constructs selective-recomputation policy from resolved generated
artifact dependencies. It forces the selected step and transitive consumers, artifact dependencies. It forces the selected step and transitive consumers,
@@ -97,7 +108,7 @@ owns the operator workflow and stable reason-code meanings.
`internal/core/debugbundle` allocates an explicitly requested per-run bundle `internal/core/debugbundle` allocates an explicitly requested per-run bundle
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command, with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
resolution, run, warning, and failure artifacts. `internal/framework/debug` resolution, run, final grouped diagnostic, and failure artifacts. `internal/framework/debug`
implements the pipeline-facing trace recorder under the trace root. implements the pipeline-facing trace recorder under the trace root.
The CLI allocates a bundle before pipeline resolution and treats requested The CLI allocates a bundle before pipeline resolution and treats requested

View File

@@ -50,11 +50,45 @@ directory selects a per-user root:
The field definitions and configuration examples are in [Configuration](config.md). The field definitions and configuration examples are in [Configuration](config.md).
On supported Unix systems, output directories and files are created with On supported Unix systems, output directories and files are created with
requested modes **0755** and **0644**. Chunk-plan, checkpoint, and debug requested modes **0755** and **0644**; debug directories and files use
directories and files use **0700** and **0600**. The operating system's umask **0700** and **0600**. The operating system's umask may impose stricter output
may impose stricter output modes. Cache and debug roots may contain sensitive or debug modes. Chunk-plan and checkpoint caches instead request maximum modes
source-derived data, so provision them for one trusted account or service. An of **0777** for new directories and **0666** for new files, including atomic
output bundle can also contain source content when its JSON output enables temporary files. The kernel applies the service's umask and applicable default
ACLs when those entries are created.
## Cache Provisioning
Cache locations select where Notarius writes, not who can access them. A
private service can use a restrictive umask:
~~~sh
umask 0077
notarius run example --config /etc/notarius/config.yml --input /srv/input.txt
~~~
A trusted-group deployment can provision a setgid cache root and use a
group-oriented umask (or an equivalent default ACL):
~~~sh
install -d -m 2770 -g notarius-operators /srv/notarius/chunk-plans /srv/notarius/checkpoints
umask 0007
notarius run example --config /etc/notarius/config.yml --input /srv/input.txt
~~~
Setgid and default ACLs control inherited group ownership and access policy;
Notarius does not copy a parent directory's literal mode bits. Cache state is
reconstructible but can contain source-derived data, so operators own its
access and retention policy. Existing cache inodes retain their prior modes
after an upgrade. Recreate a cache beneath correctly provisioned parents, or
stop writers and perform a deliberate administrative migration when a
whole-tree change is required.
Cache publication remains atomic and stored records remain validated. Those
guarantees do not add cross-process locking or transactional coordination for
simultaneous writers to one checkpoint identity.
Output bundles can also contain source content when JSON output enables
evidence publication. Apply an appropriate umask and output-root access policy evidence publication. Apply an appropriate umask and output-root access policy
before enabling that option; the requested output modes alone may not be before enabling that option; the requested output modes alone may not be
suitable for transcript-bearing bundles. suitable for transcript-bearing bundles.
@@ -81,7 +115,7 @@ precedence rules are owned by [Configuration](config.md#promptkit-profiles).
Use a profile source owned by the service account, keep it readable only by Use a profile source owned by the service account, keep it readable only by
the intended operator, and supply provider credentials through the service the intended operator, and supply provider credentials through the service
environment—not in the Notarius configuration or profile YAML. The maintained environment—not in the Notarius configuration or profile YAML. The maintained
[operator profile](../examples/profiles/dnd-extraction.yml) is secret-free and [LLM profile](../examples/llm-profiles/dnd-extraction.yml) is secret-free and
can be copied as a format starting point. Validate a deployment without a can be copied as a format starting point. Validate a deployment without a
provider call or credentials: provider call or credentials:
@@ -92,11 +126,11 @@ notarius config validate --config /etc/notarius/config.yml --pipeline dnd-sessio
An unset optional `api_key_env` reaches the provider without authorization and An unset optional `api_key_env` reaches the provider without authorization and
may receive a 401 or 403 response. may receive a 401 or 403 response.
Profile paths are currently resolved from the process working directory, not Configured PromptKit profile paths resolve from the file that supplies their
from the configuration file. The complete example's surviving declaration. The complete example's
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root `./llm-profiles/dnd-extraction.yml` path is therefore relative to that example.
invocation only. Use absolute paths such as Use absolute paths such as `/etc/notarius/profiles/dnd-extraction.yml` for
`/etc/notarius/profiles/dnd-extraction.yml` for services and containers. services and containers when that is clearer for the deployment.
## Run Lifecycle ## Run Lifecycle
@@ -105,10 +139,40 @@ and resolves configuration before module preparation and source parsing. It
then performs any permitted cache lookup, executes the pipeline, and publishes then performs any permitted cache lookup, executes the pipeline, and publishes
logical output files only after a successful runner result. logical output files only after a successful runner result.
On success, the command reports the output bundle path. A warning-bearing run On success, the command reports the output bundle path. A run with actionable
still succeeds and reports its warning count on standard error. Errors and process warnings still succeeds and reports warning-group and occurrence counts
on standard error; advisory and observation findings do not produce a warning
line. Errors and
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses). their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
## Validation Retries And Terminal Outcomes
Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from
PromptKit's structural-repair calls inside one completion and from an
LLM-backed validator's own retry budget. A semantic correction rebuilds the
ordinary producer request and supplies only the latest rejected model response
plus aggregated validator guidance; it is not a conversation replay.
After the applicable budgets are exhausted, the resolved
[`validation_policy`](config.md#pipelines) determines the result. Structural
failure and semantic rejection normally fail the run; an explicit
`reject_output` records a rejection and allows unrelated work to finish. A
validator execution failure normally uses `warn_continue`, which keeps an
otherwise accepted result in the current run with `incomplete` validation
provenance. It emits one bounded warning for every validator whose execution
budget was exhausted. A corrected result that later passes validation does not
retain diagnostics from abandoned attempts.
Treat a successful process exit as a completed run, not as proof that every
candidate was fully validated. Inspect the receipt's `validation_status`,
`validation_summaries`, rejection count, and warning group and occurrence
counts when an orchestrator requires complete validation. The durable fields
and their meanings are owned by the
[run-result receipt](integrations/run-result.md) and
[published JSON output contract](integrations/json-output.md).
## Output Bundles ## Output Bundles
Each successful run receives a generated safe run identifier and writes beneath: Each successful run receives a generated safe run identifier and writes beneath:
@@ -158,7 +222,9 @@ The configured cache mode controls one invocation:
A reused plan is still materialized and validated against the current source. A reused plan is still materialized and validated against the current source.
If a prior plan no longer gives acceptable results, use a refresh run rather If a prior plan no longer gives acceptable results, use a refresh run rather
than editing cache files. Deleting a plan is recoverable but can repeat costly than editing cache files. Deleting a plan is recoverable but can repeat costly
chunking work. chunking work. A plan accepted only under incomplete validation is not
published, and a rejected cache hit falls through to ordinary generation rather
than becoming a correction candidate.
## Checkpoint Recording, Resume, And Recompute ## Checkpoint Recording, Resume, And Recompute
@@ -183,6 +249,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
runtime identities, so checkpoints created under one state are not reused by runtime identities, so checkpoints created under one state are not reused by
either of the others. either of the others.
Only accepted, completely validated chunk, extract, merge, and normalize
results are checkpointed for reuse. Rejected, structurally invalid, and
validation-incomplete producer results remain non-reusable, even when a
`warn_continue` result advanced during its original run. A resumed invocation
therefore reruns that producer rather than treating degraded state as accepted.
Checkpoint state is confined below an identity-specific path: Checkpoint state is confined below an identity-specific path:
~~~ ~~~
@@ -236,14 +308,19 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
trace/ trace/
~~~ ~~~
The summary contains redacted invocation and resolution information plus run, The summary contains redacted invocation and resolution information, including
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace `effective-config.yml` and the separately redacted resolved pipeline, plus run,
contains allowlisted application diagnostic records and can include source or final grouped diagnostic, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
derived application data. Neither surface is a cache input. Do not treat a terminal records contain bounded attempt kinds, validator outcomes, policy,
debug bundle as safe to share merely because its configuration summary is decision, PromptKit repair count, and usage; they do not contain assistant
redacted. Invocation metadata omits reasoning effort when it is inherited, responses or complete correction messages. The trace contains allowlisted
records the replacement value when one is supplied, and records an empty value application diagnostic records and can include source, model, and correction
when inherited reasoning was explicitly cleared. content. Neither surface is a cache input. Do not treat a debug bundle as safe
to share merely because its configuration summary is redacted. Invocation
metadata omits reasoning effort when it is inherited, records the replacement
value when one is supplied, and records an empty value when inherited reasoning
was explicitly cleared. It records `config_profile` only when a non-base
configuration overlay was selected.
Notarius never creates debug state without an explicit request and never Notarius never creates debug state without an explicit request and never
automatically deletes a requested bundle. If allocation succeeds, the command automatically deletes a requested bundle. If allocation succeeds, the command

View File

@@ -38,6 +38,36 @@ Notarius is contract-first without being abstraction-heavy. Interfaces and
extension points should protect demonstrated boundaries. New abstraction is not extension points should protect demonstrated boundaries. New abstraction is not
itself an architectural goal. itself an architectural goal.
## Implementation Discipline
Simplify the mechanism, not the user's choices.
**Keep implementation machinery proportionate.** Use straightforward,
idiomatic code and introduce only the machinery needed to support the requested
behavior and flexibility. Optimize for conceptual simplicity and
maintainability, not for the fewest lines, fewest options, or narrowest product
surface.
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
harden, or otherwise narrow behavior unless required by the user, an explicit
repository contract, or an unavoidable correctness constraint. Preserve
operator and caller choice where policy has not been established. If an
additional restriction appears advisable, present it as a decision rather than
silently implementing it.
**An implementation detail is not repository policy.** Tests and documentation
must not elevate incidental behavior into a durable guarantee or restriction
unless that behavior was expressly selected as part of the contract. Existing
explicit contracts remain authoritative; changing or weakening one requires a
deliberate decision rather than being inferred from this principle.
[ADR-0017](../adr/0017-defer-cache-access-policy-to-deployment.md) illustrates
these principles. Notarius previously elevated private cache creation modes
into application policy even though Unix deployments already express access
policy through directory permissions, process umask, setgid inheritance, and
default ACLs. The revised design retains application-owned correctness
boundaries while leaving deployment security policy to the operator.
## Layers And Dependency Direction ## Layers And Dependency Direction
The application boundary is the composition root and may depend on concrete The application boundary is the composition root and may depend on concrete
@@ -146,8 +176,9 @@ lanes, validators, and LLM profile: the canonical source digest selects the
plan, while the current run still applies its configured chunk validators to plan, while the current run still applies its configured chunk validators to
the materialized chunks. the materialized chunks.
The framework owns orchestration and handoff provenance. Modules return logical The framework owns orchestration, origin enrichment, aggregation, and handoff
results and warnings; they do not own CLI reporting, physical output, cache, or provenance. Modules return logical results and classified diagnostics; they do
not own CLI reporting, physical output, cache, or
debug roots, durable file placement, or checkpoint and debug lifecycle. debug roots, durable file placement, or checkpoint and debug lifecycle.
After pipeline-wide chunking, extraction uses bounded framework concurrency. After pipeline-wide chunking, extraction uses bounded framework concurrency.
@@ -159,26 +190,53 @@ may overlap. The framework must not create unbounded goroutines per lane or
chunk. chunk.
Completion timing does not choose public ordering or errors. The coordinator Completion timing does not choose public ordering or errors. The coordinator
orders accepted artifacts, warnings, rejections, checkpoint events, and orders accepted artifacts, grouped diagnostics, rejections, checkpoint events, and
framework errors by stable pipeline scope. Rejections do not cancel unrelated framework errors by stable pipeline scope. Rejections do not cancel unrelated
work. A framework error cancels derived work, prevents undispatched work from work. A framework error cancels derived work, prevents undispatched work from
starting, waits for started work, and prevents output encoding. starting, waits for started work, and prevents output encoding.
Warnings are process-only signals: configuration degradation, approved fallback,
or incomplete configured validation. Quality uncertainty and grounding findings
are advisories; successful canonicalization and cleanup are observations.
Modules choose that semantic classification, while the framework attaches
origin, aggregates groups, enforces bounds, and presents final collections.
An ordinary successful run therefore has zero warnings. See
[ADR-0015](../adr/0015-separate-process-warnings-from-quality-diagnostics.md)
for the decision rationale.
## Validation ## Validation
Validation is a framework-managed boundary around outputs from chunk, extract, Validation is a framework-managed boundary around outputs from chunk, extract,
merge, and normalize stages. Validators receive immutable stage output merge, and normalize stages. Validators receive immutable stage output and
and make an explicit whole-output decision: approve, approve with warnings, or make an explicit whole-output decision: approve, reject, fail, or skip when a
reject. runtime prerequisite is unavailable.
Typed artifact validators receive the domain value directly. Chunk validators Typed artifact validators receive the domain value directly. Chunk validators
receive source-zone chunks, while serialized validators receive immutable receive source-zone chunks, while serialized validators receive immutable
representation bytes and declared schema metadata. A validator registered for representation bytes and declared schema metadata. A validator registered for
one target or artifact kind cannot satisfy an incompatible selection. one target or artifact kind cannot satisfy an incompatible selection.
Rejection is a recorded pipeline outcome, not a framework execution error. The framework runs every applicable validator sequentially in configured order.
Validator execution failures are framework errors. Rejected output does not It aggregates rejections, exhausted validator failures, and skips before the
advance to the next stage. producer policy chooses a disposition. A completed rejection never advances.
With no rejection, an exhausted validator failure may fail the run or, under
the configured `warn_continue` policy, advance a structurally valid candidate
with explicit incomplete-validation provenance. Validators report findings;
they do not choose candidate disposition.
A completed rejection supplies a stable reason code for internal provenance
and bounded actionable correction guidance for the candidate producer. Reason
codes, validator keys, and operator-facing messages remain diagnostic data;
they are not model instructions. The framework constructs model-facing retry
text only from the semantic guidance and fails the contract rather than
inventing or truncating missing guidance.
An LLM-backed module may also request a feedback-aware retry when its own
deterministic translation or typed safety policy rejects a structurally valid
model response. It must supply model-facing guidance separately from its
reason code, operator message, and fallback diagnostics, together with the
exact `single_response_v1` candidate. A feedback-free module retry remains
valid when no exact candidate exists.
Default validator chains are production composition policy and are registered Default validator chains are production composition policy and are registered
centrally by stage and module. Configuration may replace a stage-local default, centrally by stage and module. Configuration may replace a stage-local default,
@@ -199,6 +257,17 @@ PromptKit owns bounded structural correction within one structured completion.
Notarius owns outer stage attempts, semantic validation, and acceptance policy; Notarius owns outer stage attempts, semantic validation, and acceptance policy;
the two budgets must remain separate. the two budgets must remain separate.
An LLM-backed producer can participate in semantic correction only when it
declares `single_response_v1` and returns the exact one response that directly
controlled its candidate. On an actionable validator rejection or
feedback-capable module retry, the framework rebuilds the ordinary request and
appends only the latest defective response as an `assistant` message plus one
aggregated `user` correction message. This is a fresh replacement request, not
a growing conversation. The retry budgets, terminal policy, and sensitive-data
rationale are recorded in
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md) and
[ADR-0016](../adr/0016-feedback-aware-module-requested-retries.md).
When a model selects an application entity, callers must supply a contextual When a model selects an application entity, callers must supply a contextual
selection and deterministically attach the opaque application identity whenever selection and deterministically attach the opaque application identity whenever
the selection resolves exactly. Models do not receive or reproduce opaque the selection resolves exactly. Models do not receive or reproduce opaque
@@ -222,9 +291,11 @@ beneath this mandatory application-wide scheduler.
## Configuration And Provenance ## Configuration And Provenance
Configuration loading, precedence, defaults, environment overrides, redaction, Configuration loading, precedence, defaults, redaction, and validation are
and validation are centralized. Named pipeline definitions make structural centralized through the ConfigKit-backed boundary. `NOTARIUS_CONFIG` is limited
composition explicit and discoverable. Operational overrides are permitted to discovery; field-level environment overlays are not configuration input.
Named pipeline definitions make structural composition explicit and discoverable.
Explicit command overrides are permitted
when they do not obscure the configured pipeline structure. when they do not obscure the configured pipeline structure.
Run preparation fails before stage execution when statically discoverable Run preparation fails before stage execution when statically discoverable
@@ -233,7 +304,8 @@ invalid or incompatible.
Run manifests record enough resolved pipeline, module, source, reference, and Run manifests record enough resolved pipeline, module, source, reference, and
LLM provenance to make a run auditable after configuration changes. Manifests LLM provenance to make a run auditable after configuration changes. Manifests
record identities and summaries rather than secret or large payload content. record identities and bounded validation summaries rather than secret, raw
model, correction, or large payload content.
## State, Output, And Safety ## State, Output, And Safety
@@ -253,12 +325,32 @@ an invocation that explicitly requests resume. Debug is never a cache input and
is never created without an explicit request. Pipeline modules receive is never created without an explicit request. Pipeline modules receive
collaborator interfaces and never physical roots. collaborator interfaces and never physical roots.
Deployments own cache access policy. Cache writers request conventional maximum
creation modes and let the process umask, setgid inheritance, and default ACLs
determine effective access. Notarius owns the separate safety boundary:
confined paths, symlink rejection, atomic publication, validation, and
recovery. It does not chmod, chown, chgrp, or migrate cache entries to impose
an application policy. This rule applies to chunk-plan and checkpoint cache
only; output and debug retain their separate contracts. [Operations](../operations.md)
owns deployment provisioning and retention guidance.
Only accepted, completely validated producer output is reusable checkpoint or
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
results cannot become cache or checkpoint inputs, even when a
`warn_continue` result is allowed to advance in the current run. This
ineligibility follows derived merge and normalize results and generated
references for the remainder of the run: current-run handoff remains allowed,
but no dependent cache or checkpoint may be loaded or published.
Writes are atomic where practical. Paths for writes, moves, overwrites, and Writes are atomic where practical. Paths for writes, moves, overwrites, and
deletion must be narrow and explicit. Notarius never automatically deletes deletion must be narrow and explicit. Notarius never automatically deletes
output or requested debug bundles; cache cleanup is explicit and recoverable. output or requested debug bundles; cache cleanup is explicit and recoverable.
Secrets must not appear in errors, logs, output, cache, debug summaries, Secrets must not appear in errors, logs, output, cache, debug summaries,
traces, manifests, documentation, examples, or redacted configuration. Debug traces, manifests, documentation, examples, or redacted configuration. Raw
assistant responses and complete correction messages are attempt-local and are
excluded from ordinary durable records and summaries; the requested detailed
debug trace is the sole diagnostic surface allowed to retain them. Debug
collection is allowlisted to application-owned payloads and must not capture collection is allowlisted to application-owned payloads and must not capture
unrelated process environment values or filesystem content. Trace data may unrelated process environment values or filesystem content. Trace data may
contain application data and therefore inherits its sensitivity; operators own contain application data and therefore inherits its sensitivity; operators own

View File

@@ -30,6 +30,18 @@ that the behavior exists. The roadmap owns implementation status and sequencing
until the decision is implemented. Current architecture, user, operator, until the decision is implemented. Current architecture, user, operator,
integration, and internal documentation are updated when the behavior lands. integration, and internal documentation are updated when the behavior lands.
### Implementation Details And Policy
Documentation must distinguish an explicit contract from an incidental
implementation choice. Do not describe a restriction, guarantee, security
policy, default, or limit as authoritative merely because the current code
implements it. Documentation may accurately describe current behavior without
making it a permanent requirement; distinguish descriptive behavior from
normative policy. The
[implementation-discipline rules](architecture.md#implementation-discipline)
own the criteria for adopting product restrictions and for elevating behavior
into repository policy.
### Audience And Detail ### Audience And Detail
Write for the document's stated audience and include only the detail needed for Write for the document's stated audience and include only the detail needed for

View File

@@ -87,6 +87,13 @@ Do not force all behavior through oversized end-to-end tests. Do not test every
A test should protect a decision, contract, or invariant—not memorialize the current implementation. A test should protect a decision, contract, or invariant—not memorialize the current implementation.
Tests must not create a stronger product contract than the requirement they
protect. In particular, do not turn an incidental limit, restriction,
permission choice, validation rule, or implementation default into policy by
asserting it without an expressly selected contract. Follow the
[implementation-discipline rules](architecture.md#implementation-discipline)
when deciding what behavior deserves durable protection.
Before adding or retaining a test, ask: Before adding or retaining a test, ask:
> What realistic defect would this test catch? > What realistic defect would this test catch?

83
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,83 @@
# Notarius v0.4.0
This release strengthens LLM reliability and validation throughout the
configured pipeline, upgrades the PromptKit integration, and establishes the
source-release and downstream-consumer workflows needed for broader D&D
pipeline integration.
## Summary
Notarius now distinguishes PromptKit structural-output repair from
application-owned semantic validation retries. Producer candidates can run
through complete deterministic validator chains, receive bounded semantic
correction guidance, and retry under explicit stage policies. Final run
receipts and manifests preserve bounded validation provenance, while outputs
that advance with incomplete validation remain available to the current run
without entering reusable checkpoint state.
The release also adds a maintained complete D&D subprocess-consumer workflow,
diagnostic build versions, and the source-only release procedure used to
publish this version.
## Compatibility
- Configuration files must use schema version 4. Version 3 is not decoded or
rewritten; rename the top-level `scriptorium` section to `promptkit` when
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
format and may use its profile-inheritance support. Notarius continues to
resolve operator profiles before embedded fallbacks.
- Structural-output repair and semantic stage retries are separate bounded
mechanisms. Maintained production prompts request one structural repair by
default; explicit configuration can override the supported repair count.
- Validation policy can now fail a run, reject an output, or permit an
otherwise valid candidate to advance with incomplete-validation provenance.
The application defaults are documented in
[Configuration](../config.md#pipelines).
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
optional validation summaries plus a required validation-status field.
Consumers of this pre-release contract should follow the current
[run-result receipt](../integrations/run-result.md).
- Existing D&D artifact schema identities remain unchanged. Validation and
producer-policy changes can nevertheless cause previously accepted weak
candidates to retry, reject, or fail instead.
## Upgrade
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
to `promptkit`.
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
and ensure their credential environment variables are available at run
time.
3. Run `notarius config validate --config <path> --pipeline <id>` before the
first production invocation.
4. Review `structured_output_repair_attempts`, producer retry counts, and
`validation_policy` wherever the deployment needs behavior different from
the documented defaults.
5. Update subprocess consumers to inspect receipt `validation_status` and to
tolerate the optional bounded `validation_summaries` field. A consumer that
requires fully validated artifacts should require `approved`.
## Changes
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
inheritance, structured-output repair, typed error classification, and the
correction-aware completion protocol.
- Added pipeline and binding configuration for structural repair and terminal
validation policy, with strict startup validation and effective-setting
provenance.
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
semantic reconciliation producers. Retry prompts contain the exact defective
response and actionable semantic correction guidance without exposing
internal reason codes or opaque entity identifiers.
- Added complete validator-chain execution, validator retry handling, bounded
warnings, terminal dispositions, and durable validation summaries.
- Prevented validation-incomplete artifacts and all derived lineage from
loading or publishing reusable checkpoints while preserving same-run
generated-reference handoff.
- Tightened cached chunk-plan validation so only completely validated plans are
reused or replace stored plans.
- Added a machine-readable subprocess receipt workflow and complete D&D
consumer documentation covering all maintained artifacts.
- Added source-release checks, immutable lightweight-tag guidance, Linux and
Darwin build verification, and diagnostic `notarius --version` output.

85
docs/releases/v0.5.0.md Normal file
View File

@@ -0,0 +1,85 @@
# Notarius v0.5.0
This release separates actionable process warnings from extraction-quality
advisories and routine normalization observations, giving operators a quiet
warning channel without discarding durable diagnostic detail.
## Summary
Notarius now carries one validated, origin-aware diagnostic contract from
producers and validators through retries, reusable state, output publication,
debug summaries, run receipts, and CLI presentation. Warnings are reserved for
process degradation or incomplete configured work. Data-quality findings are
advisories, and successful deterministic cleanup is recorded as observations.
An ordinary successful run therefore reports zero warnings while retaining
bounded diagnostic provenance for later review.
The framework aggregates findings deterministically by their stable identity
and complete pipeline origin, preserves exact occurrence counts, and retains
bounded representative samples. Warning groups fail rather than truncate;
advisory and observation representation is bounded with explicit truncation
metadata and exact unrepresented-occurrence counts.
## Compatibility
- `warnings.json` now uses the incompatible grouped
`notarius.warnings.v2` envelope and contains process warnings only. Consumers
of the former flat warning payload must migrate to the current
[JSON output contract](../integrations/json-output.md).
- The new `diagnostics.json` file uses `notarius.diagnostics.v1` and contains
advisory and observation groups. Production `index.json` files always expose
both `warnings_file` and `diagnostics_file`.
- The machine-readable run receipt is now `notarius.run-result.v2`. It replaces
`warning_count` with exact warning group and occurrence counts and adds
advisory/observation group, occurrence, and truncation fields. See the
current [run-result receipt](../integrations/run-result.md).
- Custom output modules must return their complete logical file set or an
error. The former `OutputResult.Warnings` field has been removed; an output
module cannot report a warning after serializing its output.
- Reusable state now uses `notarius.workspace.v4` and chunk-plan records use
`notarius.chunk-plan.v3` so they can preserve structured diagnostics. Older
pre-release reusable state is not reused under these contracts; start with
clean state when deterministic continuity with an older workspace is not
required.
- Validation acceptance, semantic retry budgets, rejection policy, and D&D
artifact schema identities are unchanged by this release.
## Upgrade
1. Update subprocess consumers to require `notarius.run-result.v2` and read
`warning_group_count`, `warning_occurrence_count`,
`diagnostic_group_count`, `diagnostic_occurrence_count`, and
`diagnostics_truncated`.
2. Update output-bundle consumers to decode `notarius.warnings.v2`, discover
`diagnostics.json` through `index.json`, and treat diagnostics as review
information rather than process warnings.
3. Update any custom output module for the removal of
`OutputResult.Warnings`; return an error when encoding cannot complete.
4. Clear pre-release reusable state before the first upgraded production run
when deterministic continuity with an older workspace is not required.
5. Run `notarius config validate --config <path> --pipeline <id>` and perform
one representative run before promoting the release in an automated
pipeline.
## Changes
- Added validated diagnostic dispositions, categories, origins, stable reason
codes, exact occurrence counts, and bounded representative samples.
- Added deterministic run-level aggregation with separate limits for
actionable warning groups and advisory/observation groups.
- Reclassified D&D source-relatedness and unresolved-identity findings as
data-quality advisories and routine normalization changes as observations.
- Preserved structured diagnostics across producer retries, validation,
generated-reference handoff, checkpoints, chunk-plan reuse, and debug
summaries while discarding superseded-attempt findings.
- Added grouped `warnings.json`, a new grouped `diagnostics.json`, and the
corresponding production index entries.
- Upgraded the machine-readable run receipt and human CLI summary to report
exact warning and diagnostic counts without allowing advisory volume to
create warning output.
- Removed post-encoding output warnings and hardened diagnostic validation,
overflow handling, aggregate memory bounds, and warning-file path
presentation.
- Documented diagnostic ownership, classification, operator interpretation,
durable contracts, and architectural invariants in ADR-0015 and the
canonical CLI, operations, integration, and internal documentation.

84
docs/releases/v0.6.0.md Normal file
View File

@@ -0,0 +1,84 @@
# Notarius v0.6.0
This release improves the reliability and ergonomics of unattended,
subprocess-driven D&D extraction pipelines.
## Summary
Notarius now gives models more precise, semantically useful correction guidance
when deterministic or model-backed validation rejects an otherwise structured
candidate. Semantic reconciliation retries identify candidates with compact,
request-local numbers, preserve valid candidates when a proposal cannot be
repaired, and report bounded process warnings when reconciliation is
incomplete. D&D extraction also canonicalizes safely resolvable reversed source
ranges and applies more consistent schema, prompt, and validator constraints,
improving successful operation with smaller models without weakening evidence
validation.
An optional LLM-backed combat-semantics validator can review whether scene
descriptions correctly consolidate combat and apply the `combat` kind. For
subprocess callers, command-line references now follow the pipeline-scoped
configuration model: a shared reference can be supplied once and automatically
reach every compatible selected target, while lane- and binding-specific forms
remain available for exceptions.
## Compatibility
- The meaning of an unqualified `--reference slot=path` or
`--without-reference slot` selector has changed. It now applies to every
selected pipeline target that declares the slot instead of requiring exactly
one matching target. Review callers that relied on ambiguity rejection or a
unique implicit target; the current selector contract is documented in the
[CLI reference](../cli.md).
- The stage-wide `merge.slot` CLI reference shorthand has been removed. Use
`lane.merge.slot` for an exact merge binding, `lane.slot` for all compatible
bindings in one lane, or an unqualified `slot` for pipeline scope.
- Command-line file references cannot replace, remove, or coexist with a
generated artifact handoff for the same concrete target and slot. Resolution
reports the conflict so the caller can narrow or remove the selector.
- The private LLM-facing semantic-reconciliation contract now uses contiguous
`candidate_number` values rather than application identities. This is not a
durable or operator-configurable contract and requires no operator action.
- Durable D&D artifact schemas, the `notarius.run-result.v2` receipt,
`notarius.warnings.v2`, and `notarius.diagnostics.v1` are unchanged from
`v0.5.0`.
## Upgrade
1. Update subprocess commands to provide shared inputs once with
`--reference slot=path`, and convert any stage-wide `merge.slot` selector to
an explicit supported scope. The [subprocess consumer guide](../consumers/subprocess.md)
and [complete D&D consumer guide](../consumers/dnd-pipeline.md) show the
current invocation pattern.
2. Review CLI reference overrides that overlap generated same-run references;
narrow or remove an external selector rather than attempting to replace the
generated handoff.
3. Run `notarius config validate --config <path> --pipeline <id>` for every
deployed pipeline configuration.
4. Optionally add
`extract/dnd/scene-descriptions/combat_semantics` to the scene-description
validator chain where the deployment wants LLM-backed combat-scene review;
see the [configuration reference](../config.md).
5. Perform a representative run with the deployed model profile and inspect
the machine-readable receipt, warnings, and diagnostics before promotion.
## Changes
- Added feedback-aware, module-requested retry support for normalizers while
keeping retry budgets bounded and preserving attempt diagnostics.
- Reworked semantic reconciliation around compact, contiguous candidate
numbers, explicit namespace guidance, complete proposal validation, and
model-facing retry feedback that omits opaque internal identities.
- Preserved candidates separately when an exhausted semantic proposal cannot
be safely applied, with a bounded process warning describing the fallback.
- Canonicalized safely resolvable reversed D&D source-reference endpoints
before deterministic coverage validation and clarified the shared evidence
prompt.
- Improved D&D schemas, prompts, and validators with closed-value constraints,
contextual correction guidance, shared source-range coverage logic, and
clearer item-holder transition rules.
- Added the optional D&D scene-description combat-semantics validator, shared
combat policy assets, evaluation fixtures, and retry-path coverage.
- Made unqualified CLI references pipeline-scoped, added hierarchical lane and
exact-binding selectors, defined deterministic override precedence, and
documented first-class subprocess use.

88
docs/releases/v0.7.0.md Normal file
View File

@@ -0,0 +1,88 @@
# Notarius v0.7.0
This release simplifies configuration management with ConfigKit and makes
cache access permissions a deployment responsibility.
## Summary
Notarius adopts ConfigKit v0.3.0 for strict configuration loading, imports,
configuration profiles, provenance, and redacted inspection. Operators can
compose whole pipeline definitions from separate files and select development
overlays without duplicating the base configuration.
Configured relative filesystem paths now resolve against their declaring
files, including imported definitions and profile replacements. The effective
configuration and its explanation reflect the paths used at runtime.
## Compatibility
- Configuration version 5 replaces version 4; there is no compatibility loader.
Numeric `version: 5` is the canonical spelling; quoted `"5"` also works.
- Module and validator bindings require objects with `module:`; external
references require objects with `path:`. Lane-level reference aliases are
removed; use binding-local references instead.
- Strict ConfigKit YAML parsing rejects aliases, merge keys, and unsupported
scalar forms. See the [configuration reference](../config.md) for the
complete contract and migration guidance.
- The seven operational environment overrides are removed:
`NOTARIUS_TOTAL_LLM_CONCURRENCY`, `NOTARIUS_STAGE_WORKERS_EXTRACT`,
`NOTARIUS_OUTPUT_DIR`, `NOTARIUS_CACHE_CHUNK_PLANS_MODE`,
`NOTARIUS_CACHE_CHUNK_PLANS_DIR`, `NOTARIUS_CACHE_CHECKPOINTS_DIR`, and
`NOTARIUS_DEBUG_DIR`. `NOTARIUS_CONFIG` still selects the configuration
file; PromptKit credential environment variables remain supported.
- Configured relative reference, PromptKit profile, output, cache, and debug
paths use the canonical declaring file's directory. Built-in relative
defaults use the main configuration directory. CLI-relative paths still use
the invocation directory; absolute paths are unchanged.
- Requested debug bundles now contain `effective-config.yml` instead of
`effective-config.json`. Inspection output is redacted and is not promised
to be reloadable configuration.
- Chunk-plan and checkpoint cache writers respect deployment permissions
through ordinary creation modes, umask, setgid inheritance, and default ACLs;
they no longer impose private permissions. Existing entries are not chmodded
or migrated. Output and debug access contracts are unchanged.
- Durable extraction artifact schemas and subprocess receipt formats are
unchanged by this release.
## Upgrade
1. Migrate deployed configurations to version 5 using the
[configuration migration guide](../config.md#version-5-migration). Expand
shorthand bindings and references, and move retired environment settings
into YAML or supported CLI overrides.
2. Review relative paths, particularly output/cache/debug directories and
PromptKit profile sources. Use absolute paths when preserving an existing
deployment location is important.
3. Review deployment access settings for cache directories. Provision umask,
groups, and ACLs as appropriate; Notarius does not choose the installation's
access policy. See [Operations](../operations.md).
4. Use `config validate --config <path> --pipeline <id>` for each pipeline,
including selected configuration profiles. Inspect resolved settings with
`config effective` and their provenance with `config explain`; see the
[CLI reference](../cli.md).
5. Update any debug-bundle tooling that consumes the former effective JSON
file. Subprocess callers may optionally select an overlay through
`--config-profile`; this is separate from PromptKit's `--llm-profile`.
The maintained [minimal](../../examples/dnd-minimal.config.yml) and
[complete](../../examples/dnd-complete.config.yml) D&D examples demonstrate
self-contained configuration and composed pipelines with a development profile.
Build from the source tag using the [source-release procedure](../release.md).
## Changes
- Replaced custom YAML parsing and environment-overlay machinery with a
ConfigKit-backed typed document boundary and detached runtime conversion.
- Added imports, configuration profiles, source-aware normalization, safe
effective/explanation commands, and source-file legends for load errors.
- Preserved explicit zeroes and empty validator overrides at supported stages,
rejected invalid reference branches before conversion, and retained
whitespace normalization for worker identifiers.
- Corrected normalization recording for ordered steps and validator sequences,
with regression coverage for runtime values and provenance.
- Accepted ConfigKit's integer representation at module-option boundaries
through checked shared conversion.
- Updated examples, configuration and subprocess guidance, and debug rendering.
- Reworked cache creation to respect deployment-selected permissions while
retaining confined paths, symlink rejection, atomic publication, validation,
and recovery.

44
docs/releases/v0.7.1.md Normal file
View File

@@ -0,0 +1,44 @@
# Notarius v0.7.1
This release updates the prompt engine and its provider catalogs while retaining
Notarius's application-owned D&D defaults and subprocess contracts.
## Summary
- Upgrade PromptKit from v0.9.0 to v0.10.0 and adopt its selected provider catalogs.
- Refresh the profile-source checkpoint identity for the updated catalog.
- Document profile ownership and upstream catalog compatibility.
## Compatibility
Notarius's CLI, configuration schema, durable artifact formats, and subprocess
receipt are unchanged. The `dnd-extraction` fallback retains its explicit model
and settings; the upgrade does not switch it to an upstream default profile.
Upstream catalog selections require review: `deepseek-3-2`, `deepseek-4-flash`,
and `deepseek-4-pro` are no longer present. Operators selecting or inheriting
those profiles must supply a complete operator-owned definition or select an
available profile. `deepseek-flash-latest` and `deepseek-pro-latest` are not
equivalent aliases and need not preserve previous settings. See the
[PromptKit integration contract](../integrations/pkg-promptkit.md#profile-sources-and-compatibility).
Existing checkpoints with the previous profile-source identity are not reused.
Source-keyed chunk plans and existing output artifacts are unaffected.
## Upgrade
Review selected LLM profile IDs and inherited bases before rebuilding from this
tag. No Notarius configuration migration is required for the maintained examples
or application-owned D&D defaults. For controlled model settings, use a complete
[operator profile](../config.md#promptkit-profiles), rather than inheriting a
volatile catalog entry. Follow the [source release procedure](../release.md)
for source installation and verification.
## Changes
- PromptKit v0.10.0 preserves the public engine API and adds two opt-in maintained
profile names; Notarius continues using its existing adapter and defaults.
- The dependency graph selects OpenRouter catalog v2.20260922.1 and Rakestrawhome
catalog v2.20260922.0.
- Profile checkpoint fingerprinting and current integration documentation now
identify the updated engine.

View File

@@ -0,0 +1,648 @@
# Warning Signal And Presentation Audit
## Executive Assessment
Notarius warning execution is mechanically stronger than its operator-facing
presentation. Terminal-attempt promotion, stable ordering after concurrent
work, checkpoint replay, validation summaries, and debug retention are all
substantially correct. The audit found no general duplicate-append defect in
the extract, merge, or normalize handoffs and no leakage of abandoned-attempt
warnings into a successful result.
The warning channel itself is not coherent. One flat `contracts.Warning` type
currently represents at least four materially different concepts:
- actionable degradation or incomplete validation;
- heuristic data-quality doubt;
- successful but potentially reviewable fallback; and
- routine canonicalization, ordering, and deduplication observations.
That conflation is the primary reason successful runs produce a count that is
large but operationally weak. The maintained complete D&D example demonstrates
the problem without a live provider: an approved run with no rejected outputs
published 12 warning records, all from three advisory relatedness checks. An
operator separately reported a successful complete D&D run with 10 outputs,
one rejection, and 85 warnings. The production bundle for that run was not
available in this environment, so its reason-code distribution could not be
measured.
The current implementation also has four correctness or robustness gaps:
1. warning records lose stage, step, lane, module, validator, and chunk
provenance when promoted, which makes safe aggregation and diagnosis
impossible from `warnings.json` alone;
2. there is no framework-level validation or aggregate bound, and the NPC- and
spell-relatedness validators bypass the D&D warning limiter entirely;
3. warnings returned by an output encoder are added after `warnings.json` has
already been encoded, so the receipt, stderr, debug bundle, and published
warning file can disagree; and
4. a skipped validator contributes to `incomplete` validation but does not
receive the warning generated for an exhausted validator failure.
The recommended end state is a structured diagnostic contract with explicit
disposition, category, origin, occurrence count, and bounded samples. Warnings
are reserved for process-level degradation or incompleteness. LLM-judged or
deterministically inferred extraction-quality signals are advisories, never
warnings, and routine normalization observations remain inspectable without
being reported as top-level warnings. An ordinary successful run in which all
configured work completes normally should therefore report zero warnings. This
is an architectural and durable-contract change, not merely revised CLI prose.
## Evidence And Limits
The audit used:
- a complete static search of production `contracts.Warning` constructors,
reason-code constants, result fields, and promotion sites under `internal/`;
- call-path inspection through producer attempts, validators, lane
coordination, chunk-plan reuse, checkpoints, output encoding, debug output,
CLI presentation, and run-result construction;
- the maintained complete and minimal D&D examples with offline fake LLMs;
- focused deterministic tests for warning bounds, semantic-reconciliation
fallback, `warn_continue`, semantic retries, terminal rejection, concurrency
ordering, and checkpoint reuse; and
- the operator-provided observation of an 85-warning complete D&D run.
No provider-backed production run was attempted because this environment has
no API key. Consequently, the audit can establish warning mechanics, possible
multiplicity, synthetic volume, and obvious heuristic limitations, but cannot
estimate production frequency or the real false-positive rate of individual
D&D advisories. Those measurements are not required to choose the recommended
architecture; they are required before strengthening any heuristic advisory
into a rejection or setting a numerical production acceptance target.
## Complete Warning-Producer Inventory
### Framework And Generic Boundaries
| Producer | Reason code | Trigger and consequence | Multiplicity and bound | Current surfaces and coverage |
| --- | --- | --- | --- | --- |
| Reference materialization in `internal/framework/pipeline/references.go` | `empty_reference` | A bound external reference is a valid, accepted media type but contains zero bytes. The prompt may receive materially incomplete context. | One per empty bound file; finite by configuration but no shared run-level cap. | Enters `RunInput.Warnings`; reference tests protect contextual scope. |
| Producer-attempt policy in `internal/framework/pipeline/producer_attempts.go` | `validator_execution_incomplete` | An applicable validator exhausted its execution budget and `warn_continue` accepted the otherwise valid candidate. | One per failed validator on each terminal candidate. An extract chain can multiply this by chunks and lanes. There is no global cap. | Durable warning, receipt count, stderr, debug, and validation summary. `TestWarnContinueRecordsOneWarningForEachExhaustedValidator` covers failures. |
| Chunk, extract, merge, normalize, and output module result contracts | Module-defined | A module may return arbitrary warnings with its successful candidate. | No contract validation, message limit, per-result cap, or global cap. Current production modules are inventoried below. | Terminal-attempt filtering and concurrency ordering are well tested. |
| Production JSON output encoder | None | The encoder copies incoming warnings into `warnings.json`; it does not currently create warnings. | Same count as its input. | JSON encoder and assembled-pipeline tests compare the incoming run warnings with the published file. |
| Output encoder result contract | Module-defined | Any output encoder may return warnings discovered during encoding. | Unbounded by contract. No production encoder currently exercises this capability. | Appended to final `RunOutput` only after logical files were encoded; this is the cross-surface defect described in AUD-WARN-004. |
Input adapters and production mergers do not currently have independent
warning producers. Chunk-plan and checkpoint decisions are structured manifest
or debug provenance rather than warnings. Cancellation and hard persistence,
reference, parsing, serialization, and provider failures remain errors.
### D&D Extraction Gates
| Producer | Reason code | Trigger and consequence | Multiplicity and bound |
| --- | --- | --- | --- |
| `dnd/combat-turns` extractor | `scene_classification_unavailable` | The chunk has no exact matching scene-description classification. The extractor returns an empty accepted result and skips the LLM, so combat-turn output may be incomplete. | At most one per chunk for this lane. |
| `dnd/enemy-events` extractor | `scene_classification_unavailable` | The same missing or mismatched scene gate causes accepted empty enemy-event output. | At most one per chunk for this lane. |
An exact non-combat classification produces an intentional empty result without
a warning. An exact combat classification proceeds normally. The two producers
share a code and operator consequence but use different messages; their module
origins are not retained in the final warning record.
### D&D Source-Relatedness Validators
All ten relatedness validators are deterministic advisories: they approve the
candidate and warn when contextual prose or an entity name is not lexically
present in cited text. Shape and source-reference failures are deliberately
left to blocking validators earlier in the chain. The same relatedness
validator is registered in both the extract and normalize default chain for
each artifact family in `internal/modules/dnd/register/chains.go`.
| Artifact family | Warning reason | Per-record trigger | Local bound | Omission reason |
| --- | --- | --- | --- | --- |
| Combat turns | `combat_turn_not_near_source` | Actor token sequence absent | 20 per validator invocation | `combat_turn_relatedness_warnings_omitted` |
| Enemy events | `enemy_event_not_near_source` | Subject token sequence absent | 20 | `enemy_event_relatedness_warnings_omitted` |
| Item occurrences | `item_occurrence_source_unrelated` | Item name token sequence absent | 20 | `item_occurrence_relatedness_warnings_omitted` |
| Item registry | `item_not_near_source` | Item name token sequence absent | 20 | `item_relatedness_warnings_omitted` |
| Location occurrences | `location_occurrence_not_near_source` | Location name token sequence absent | 20 | `location_occurrence_relatedness_warnings_omitted` |
| Location registry | `location_not_near_source` | Location name token sequence absent | 20 | `location_relatedness_warnings_omitted` |
| NPC occurrences | `npc_occurrence_not_near_source` | NPC name token sequence absent | 20 | `npc_occurrence_relatedness_warnings_omitted` |
| NPC registry | `npc_not_near_source` | NPC name token sequence absent | **Unbounded** | None |
| Scene descriptions | `scene_description_not_near_source` | No significant title or summary token appears; up to two findings per scene | 20 | `scene_description_relatedness_warnings_omitted` |
| Spells | `spell_not_near_source` | Spell-name token sequence absent | **Unbounded** | None |
The eight limiter-generated omission records are presentation artifacts, not
new source-relatedness conditions. They occupy a warning slot and make list
length differ from the actual occurrence count.
### D&D Normalizers
Every production D&D normalizer bounds its returned warning slice to 20 through
`internal/modules/dnd/shared/diagnostics`, including a final omission record
when needed. Registry semantic retries reserve space for their fallback
warning. The following table is complete by semantically distinct condition;
codes listed together are parallel artifact-family variants.
| Condition | Reason codes | Result impact | Current classification assessment |
| --- | --- | --- | --- |
| Display or field whitespace/name canonicalization | `npc_fields_normalized`, `item_fields_normalized`, `location_fields_normalized`, `spell_name_canonicalized`, `combat_actor_canonicalized`, `enemy_event_name_canonicalized`, `item_occurrence_name_canonicalized`, `location_occurrence_name_canonicalized`, `scene_description_prose_normalized` | Deterministic successful mutation. The item-occurrence code can also describe `from`/`to` whitespace, not only the item name. | Routine observation. |
| Durable ID recomputation | `npc_id_recomputed`, `item_id_recomputed`, `location_id_recomputed` | Restores the deterministic name-derived ID. | Routine observation; invalid identity is separately rejected by default chains. |
| Source-reference sorting or deduplication | `source_references_normalized` | Sorts and removes exact duplicate references while deliberately preserving invalid references for their validators. | Routine observation. Shared code is useful but ambiguous without producer origin. |
| Canonical record ordering | `combat_turns_reordered`, `enemy_events_reordered`, `item_occurrences_reordered`, `location_occurrences_reordered`, `npc_occurrences_reordered`, `scene_description_order_normalized` | Deterministic order changes only. | Routine observation. |
| Exact or approved semantic duplicate consolidation | `duplicate_npc_collapsed`, `duplicate_item_collapsed`, `duplicate_location_collapsed`, `duplicate_spell_cast_collapsed`, `duplicate_combat_turn_collapsed`, `duplicate_enemy_event_collapsed`, `duplicate_item_occurrence_collapsed`, `duplicate_location_occurrence_collapsed`, `duplicate_npc_occurrence_collapsed`, `scene_description_duplicate_collapsed` | Removes duplicate records and preserves or combines canonical evidence according to the artifact policy. Registry codes cover both exact and accepted semantic consolidation. | Durable normalization observation; not normally operator-actionable. |
| Unresolved external membership | `spell_name_unresolved`, `item_occurrence_unknown_item_id`, `location_occurrence_unknown_location_id` | The value is preserved but is not grounded in the effective catalog or registry. Default chains normally reject the same condition before normalization; it remains reachable with validator overrides or defensive direct use. | Actionable data-quality warning. |
| Unsafe currency consolidation proposal | `item_semantic_proposal_invalid` | The proposed group is rejected and all records are preserved because denominations or currency/non-currency members are incompatible. The same code is also used internally as a retry reason. | Advisory about model proposal quality; no accepted-data loss. The control and diagnostic meanings should be separated. |
| Semantic reconciliation unavailable or exhausted | `npc_semantic_reconciliation_exhausted`, `item_semantic_reconciliation_exhausted`, `location_semantic_reconciliation_exhausted` | The safe deterministic result is accepted, but possible semantic duplicates remain. | Actionable fallback warning. |
| Local warning truncation | `npc_normalization_warnings_omitted`, `item_normalization_warnings_omitted`, `location_normalization_warnings_omitted`, `spell_normalization_warnings_omitted`, `combat_turn_normalization_warnings_omitted`, `enemy_event_normalization_warnings_omitted`, `item_occurrence_normalization_warnings_omitted`, `location_occurrence_normalization_warnings_omitted`, `npc_occurrence_normalization_warnings_omitted`, `scene_description_normalization_warnings_omitted` | Reports that individual records were omitted from presentation. | Group metadata, not an independent warning. |
No production D&D normalization warning exposes raw model responses,
correction guidance, or provider errors. Most dynamic names are quoted and
truncated by the shared helper. That local discipline is not enforced by the
generic warning contract, and the spell relatedness message does not use the
shared truncation helper.
## Warning Propagation And Surface Map
```text
external-reference warnings -----------------------------+
|
module candidate warnings -> validation chain warnings |
| | |
+---- producer-attempt terminal policy -------+
| |
accepted / terminal rejection only |
| |
chunk or lane result in canonical order |
| |
checkpoint record/replay and ordered step merge |
| |
RunOutput.Warnings <------------+
|
OutputRequest -> output encoder
| |
warnings.json OutputResult.Warnings
|
appended to final RunOutput only
|
receipt, stderr, final debug warning summary
```
### Attempts And Validation
- `runProducerAttempts` promotes only the terminal accepted or terminal
rejected candidate's module and completed-validator warnings. Operational,
structural, semantic, and module-directed attempts that are superseded are
retained in attempt debug artifacts but not in the final collection.
- A module-directed semantic-reconciliation retry adds its fallback warning
only when no retry remains. Earlier attempt warnings are discarded.
- On `warn_continue`, warnings from the otherwise accepted candidate and
completed approved or rejected validators are retained. One fixed,
non-sensitive `validator_execution_incomplete` warning is added for every
failed validator. Skipped validators affect the validation summary and final
`incomplete` status but do not receive such a warning.
- A semantic terminal rejection retains only warnings from that rejected
attempt. Structural rejection after producer failure cannot retain a
candidate warning because no valid candidate result exists.
These behaviors are protected by the producer-attempt, extract-handoff,
rejection-warning, normalize-retry, and attempt-debug tests. They are the right
foundation for the redesign and should not be replaced with early-exit or
all-attempt accumulation.
### Concurrency And Ordering
Extract jobs are dispatched chunk-first and lane-second. Results are stored by
chunk index, finalized in ascending chunk order, and lane continuations are
merged into a slice indexed by configured lane order. Pipeline steps run in
configured order. The resulting public order is therefore:
1. pre-run reference warnings;
2. chunk-stage warnings;
3. step order;
4. configured lane order within each step;
5. chunk order within extract;
6. merge warnings; then
7. normalize warnings; followed by any output-result warnings.
`TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion` exercises warning
order under reversed completion. No completion-order leak was found.
### Chunk Plans And Checkpoints
- A reusable chunk plan stores producer warnings only. Current validators run
again, and their current warnings are appended. Warnings from a cached plan
candidate that fails current validation are discarded before regeneration.
- Accepted extract, merge, and normalize checkpoints store the terminal
warnings for their stage. Reuse loads and appends those warnings once at the
same logical handoff. Tests compare fresh and resumed warning collections and
preserve their order.
- Validation-incomplete accepted outputs are not reusable, preventing a later
run from silently treating incomplete validation as complete.
- Required accepted-normalize hydration replays that normalize checkpoint's
warnings; checkpoint decisions separately expose that reuse occurred.
The recommended redesign should keep fresh and resumed logical diagnostics
equivalent. Whether a result was reused belongs in checkpoint provenance, not
in the diagnostic grouping key; adding a `reused` distinction would fragment
groups and make equivalent runs present differently.
### Terminal Surfaces
| Surface | Current content | Audience | Audit result |
| --- | --- | --- | --- |
| `RunOutput.Warnings` | Flat final slice | Framework and CLI | Canonical in-memory list, but lacks origin and bounds. |
| Published `warnings.json` | Object containing the warnings passed into the output encoder | Durable consumers | Exact for the production JSON encoder unless the encoder itself returns warnings. |
| `index.json` | Path to `warnings.json` | Durable consumers | Stable discovery path; no separate diagnostic-detail path. |
| Run-result v1 | `warning_count = len(final RunOutput.Warnings)` | Subprocess callers | Count only; no group/occurrence distinction. |
| Human stderr | `run completed with N warning(s)` | Operators | Count only and no direct detail path. Successful exit remains zero. |
| Manifest | Validation and rejection summaries, no warning collection | Durable provenance | Correctly avoids duplicating the flat list. |
| Debug summary `warnings.json` | Raw final warning array | Operators/developers | Includes final output-result warnings and can therefore differ from published `warnings.json`. |
| Debug run report | Final warning count | Operators/developers | Same final slice length as receipt and stderr. |
| Attempt/stage debug | Candidate-local warning detail and origin in path/envelope | Forensics | Sufficient to diagnose provenance, but debug capture is optional and is not a durable consumer contract. |
## Empirical Measurements
### Offline And Synthetic Runs
| Scenario | Result | What it establishes |
| --- | --- | --- |
| Maintained complete D&D config and transcript with the repository's offline fake LLM | Approved, 10 normalized outputs, 0 rejected outputs, 12 warnings; receipt, stderr, and published file all reported 12 | An ordinary structurally successful workflow can be noisy without fallback or incomplete validation. |
| Same complete run, grouped after publication | Three reason codes, seven exact `(reason, scope, message)` tuples, maximum exact-tuple repetition of three | The flat count materially overstates distinct operator conditions. Scope resets within chunks and does not identify origin. |
| Maintained focused scene-description workflow | Approved, one normalized output, 0 warnings | The warning channel can be quiet when synthetic model text is lexically grounded. |
| Generic warning publication contract | One warning reaches successful stderr, durable output, and debug summary | The ordinary pre-output path is consistent. |
| NPC semantic-reconciliation candidate-limit fallback | No LLM call, all records preserved, one exhaustion warning, total warnings no greater than 20 | Fallback is bounded and materially different from routine normalization. |
| `warn_continue` with two failed validators and one skipped validator | Validation status contains all three incomplete validators; warning slice contains two execution-incomplete records | Current warning count does not describe all incomplete validation. |
| Retrying extract candidate | Two producer attempts; only the accepted attempt's one warning is final | Retry does not amplify abandoned warnings. |
| Terminal semantic rejection | Only the final rejected attempt's operation and validator warnings are final | Rejection diagnostics are retained without retaining superseded warnings. |
| Fresh versus reused extract checkpoint | Warning collections are deeply equal | Checkpoint replay does not itself amplify warnings. |
| Spell normalizer with 21 unresolved entries | 20 records: 19 samples plus one omission record saying two additional warnings were omitted | `warning_count` is neither exact occurrence count nor distinct-condition count. |
The focused audit tests passed in `internal/cli`,
`internal/framework/pipeline`, the NPC-registry and spell normalizers, and all
D&D packages.
### Bounded Sample Review
The complete offline D&D run produced:
| Reason | Count | Sample | Review |
| --- | ---: | --- | --- |
| `location_not_near_source` | 4 | `Moon Gate` was absent from cited text | Correctly identifies deliberately unsupported fake output. Three records shared the same exact tuple because chunk and stage origin were lost. |
| `location_occurrence_not_near_source` | 4 | A `Moon Gate` visit was absent from cited text | Correctly identifies the same unsupported registry-driven occurrence, but repeats the same operator concern across extraction and normalization. |
| `scene_description_not_near_source` | 4 | A title or summary had no significant exact token in cited text | Mixed value. Generic `session scene` prose is ungrounded, while `Arrival` versus transcript `arrive` illustrates an expected lexical false positive. |
This fake workflow is an integration fixture, not a model-quality benchmark.
It nonetheless proves that the checks carry useful evidence while being too
imprecise and repetitive to serve as one-warning-per-record operator alerts.
### Production Evidence Still Needed
The reported 85-warning run establishes that high volume occurs in practice,
but the following remain unknown:
- dominant production reason codes and stage/lane sources;
- unique group count versus repeated occurrence count;
- false-positive rate for each relatedness family;
- how much volume comes from normalization observations versus advisories;
- whether fresh and resumed production runs remain equivalent; and
- a defensible numerical acceptance target.
If further data is worthwhile, the operator can supply the v1 receipt,
`warnings.json`, and manifest validation summaries without supplying transcript
or lane artifacts. An initial privacy-preserving report should group by reason
code and normalized scope family, count exact repeated tuples, and omit message
text. Reviewing heuristic precision requires a separately approved bounded
sample with its cited source context.
## Classification Of Current Conditions
| Target disposition | Current families | Result impact | Operator action | Durable placement |
| --- | --- | --- | --- | --- |
| **Warning** | Empty reference; validator failure or skip accepted under `warn_continue`; unavailable required scene classification; exhausted semantic reconciliation | A configured process completed under an allowed degraded or incomplete policy rather than completing normally | Correct reference/configuration, inspect provider/validator, or rerun | Actionable `warnings.json`, receipt summary, stderr summary, debug |
| **Advisory** | Source-relatedness heuristics; unresolved spell or registry membership; guarded invalid semantic proposal; any future LLM-judged uncertainty or extraction-quality signal | Uncertain data quality or poor model proposal, but accepted data is structurally valid and deterministic guards prevented unsafe mutation | Optional model/source review; no routine action for every record | Durable diagnostic detail and debug; never a top-level warning |
| **Observation** | Whitespace/name/ID/source-reference canonicalization; canonical ordering; exact and approved semantic duplicate consolidation | Successful intended normalization | None under normal operation | Durable bounded normalization diagnostics or debug; no stderr warning |
| **Not a diagnostic** | Rejection, invalid structure, cancellation, persistence error, provider failure under fail-run policy | Candidate or run did not complete according to policy | Inspect rejection/error and retry or correct input/configuration | Existing rejection, validation summary, error, and debug contracts |
Exact and semantic duplicate consolidation should remain distinguishable in
category or reason metadata even though both are observations. Semantic
reconciliation exhaustion remains a warning because a capability was not
applied; successful approved consolidation is an observation because it is the
normalizer's intended work.
## Ranked Findings
### AUD-WARN-001 — The Flat Warning Type Destroys Signal Quality
- **Priority:** High operator impact; high implementation leverage.
- **Evidence:** `contracts.Warning` has only scope, reason, and message. Routine
normalizer changes, heuristic doubt, fallback, and incomplete validation all
enter the same slice and the same CLI count. The offline complete run's 12
records were all advisories; the operator observed 85 records in a successful
real run.
- **Impact:** Operators cannot tell whether a warning requires a rerun, a
configuration repair, optional review, or no action. Repeated routine output
trains them to ignore the channel.
- **Recommendation:** Replace the flat semantic contract with explicit
`warning`, `advisory`, and `observation` dispositions plus a small category
vocabulary. Do not infer disposition from message text or require every
downstream consumer to maintain a reason-code policy table.
### AUD-WARN-002 — Warning Records Lose The Origin Needed For Diagnosis And Aggregation
- **Priority:** High correctness and usability impact.
- **Evidence:** The runner knows stage, step, lane, module, validator, chunk ID,
and chunk index at promotion time, but `terminalWarnings` flattens module and
validator records into `[]contracts.Warning`. Per-chunk scopes such as
`locations[0]` and `occurrences[0]` then repeat without identifying their
chunk or producer. `source_references_normalized` is intentionally shared
across families and is therefore especially ambiguous.
- **Impact:** `warnings.json` cannot answer which stage or module produced a
record. Message- or scope-based deduplication would merge unrelated findings
or retain accidental duplicates.
- **Recommendation:** Keep module findings free of framework context, then have
the framework add a structured origin envelope before promotion. Validator
findings must retain validator identity instead of passing through
`validationReport.Warnings()` as a flat slice.
### AUD-WARN-003 — Warning Volume Is Not End-To-End Bounded Or Validated
- **Priority:** High robustness impact; medium immediate likelihood.
- **Evidence:** D&D's `LimitWarnings` caps most individual producers at 20, but
NPC- and spell-relatedness return one warning per record without the helper.
Every extract validator is invoked per chunk, all ten relatedness checks run
again after normalization, and there is no run-level collector. The generic
contract validates neither disposition nor reason/message size, UTF-8,
blankness, or total records.
- **Impact:** Warning memory and output grow with chunks, lanes, configured
validators, and record counts. Local omission records lose exact occurrence
semantics while still incrementing `warning_count`.
- **Recommendation:** Add a generic bounded diagnostic collector that preserves
exact occurrence counts and bounded samples. Validate all diagnostic fields
at the module/framework boundary. Immediately bring NPC and spell
relatedness under the existing cap if the full redesign is staged.
### AUD-WARN-004 — Output Encoder Warnings Make Durable Surfaces Disagree
- **Priority:** Medium current impact; high contract correctness risk.
- **Evidence:** `Runner.Run` passes existing warnings to `encoder.Encode`, then
the production encoder serializes `warnings.json`. Only after encoding does
the runner append `OutputResult.Warnings`. The receipt, stderr, debug summary,
and debug run report see the final slice; the already-created published file
cannot. No production encoder currently returns a warning, so ordinary JSON
runs do not trigger the defect.
- **Impact:** A valid output-module implementation can violate the documented
claim that `warning_count` describes the published warning collection.
- **Recommendation:** Remove successful output warnings from the output-module
contract unless a demonstrated use case requires them; encoding failures
should be errors and optional encoder observations should be debug data. A
two-phase finalize API is the viable but more complex alternative.
### AUD-WARN-005 — Validation Skips Are Incomplete But Not Warned
- **Priority:** Medium operator/correctness impact.
- **Evidence:** `firstIncompleteValidation` treats failed and skipped validators
alike, and validation summaries include both. `incompleteValidationWarnings`
emits records only for `validationFailed`. The focused test demonstrates
three incomplete validators but two warnings.
- **Impact:** A successful run can have `validation_status: incomplete` while
its warning count understates or even omits the affected validators. A caller
that checks only warnings receives a weaker signal than the manifest and
receipt status.
- **Recommendation:** Produce one aggregated incomplete-validation warning
group whose occurrences cover both failure and skip, while retaining typed
outcome and safe reason metadata in the validation summary. Do not expose
provider errors or arbitrary skip prose in model or operator messages.
### AUD-WARN-006 — Relatedness Checks Are Useful But Repetitive And Lexically Weak
- **Priority:** Medium operator impact; low acceptance-policy urgency.
- **Evidence:** Every family runs the advisory in both extract and normalize
chains. The complete fixture contains exact repeated tuples, and the checks
rely on exact normalized token sequences or a minimal significant-token
overlap. Reason naming drifts between `*_not_near_source` and
`*_source_unrelated`.
- **Impact:** The checks can catch unsupported entities, but aliases, pronouns,
inflection, and generic scene prose create predictable false positives. Flat
per-record presentation magnifies them.
- **Recommendation:** Retain the validators and their stage-local execution,
but classify and aggregate them as advisories. Normalize reason-code naming
when the diagnostic contract changes. Do not strengthen them into rejection
rules without a human-reviewed production evaluation.
### AUD-WARN-007 — `warning_count` Has No Stable Operational Meaning
- **Priority:** High downstream-contract impact.
- **Evidence:** The receipt and CLI use `len(output.Warnings)`. One list element
can be an omission summary representing several hidden occurrences; repeated
records can represent the same condition; and skipped validators can be
absent. A 21-occurrence spell test produces a list length of 20.
- **Impact:** The value is neither an exact occurrence count nor a distinct
warning-group count. Consumers cannot set policy or present a trustworthy
summary from it.
- **Recommendation:** Introduce explicit warning-group and warning-occurrence
counts in a versioned receipt. Do not silently redefine the v1 field.
## Recommended Target Contract And Presentation Model
### Diagnostic Model
Use one validated internal diagnostic model with these concepts:
- **disposition:** `warning`, `advisory`, or `observation`;
- **category:** a small enum such as `configuration`, `degradation`,
`validation_incomplete`, `data_quality`, `fallback`, or `normalization`;
- **reason code:** stable semantic identity owned by the producer;
- **origin:** framework-added phase/stage, step ID, lane ID, module key,
validator name, chunk ID, and chunk index when applicable;
- **occurrence count:** exact number of matching findings;
- **samples:** a small deterministic list of bounded scope/message pairs; and
- **omitted sample count:** `occurrence_count - len(samples)`, represented as
metadata rather than another diagnostic record.
Errors and rejected outputs must not become diagnostic dispositions. A warning
means that the run completed under policy despite a process-level degradation
or incomplete configured operation. Advisory and observation dispositions can
describe accepted artifact quality and transformation provenance, but no
LLM-judged extraction-quality signal may be promoted to a warning. Validation
status remains authoritative for approval, rejection, and incomplete
validation.
### Aggregation
The framework runner should own aggregation after it enriches findings with
origin and before public output construction. Modules and validators retain
semantic ownership of disposition, category, reason, scope, and message; they
must not own CLI or file presentation.
The default stable key should be:
```text
disposition + category + reason_code
+ phase/stage + step_id + lane_id + module_key + validator_name
```
Chunk, record scope, and message text belong in samples and must not be part of
the group key. This groups repeated per-chunk findings without merging the same
code across distinct producers or pipeline locations. Group order should be
the first occurrence in the runner's existing canonical order; sample order
should follow the same order. A final canonical sort by the complete origin key
is also viable, but completion timing must never choose either order.
Aggregation must be incremental and bounded. Producers should use a shared
collector that counts every occurrence while retaining only bounded samples;
the framework then merges producer groups without reconstructing counts from
omission prose. A global maximum group count is also required, with overflow
represented by structured aggregate metadata and with actionable groups given
priority over lower dispositions.
### Durable Files
Keep one canonical home for each class:
- `warnings.json` should contain versioned, grouped actionable warnings only;
- a new `diagnostics.json` should contain versioned advisory and observation
groups only, avoiding duplication of warning groups;
- `index.json` should link both files;
- `rejected.json` and manifest validation summaries should retain their current
separate responsibilities; and
- debug bundles should retain candidate-attempt detail plus the final grouped
projections.
This is preferable to keeping all detail in `warnings.json` and filtering only
the CLI: downstream consumers would otherwise continue to receive a semantically
mixed warning contract, and routine observations would still dominate the
durable file.
### CLI And Receipt
For a successful human run with actionable warnings, print a concise summary
such as:
```text
notarius: run completed with 2 warning groups (7 occurrences); details=/.../warnings.json
```
Advisories and observations should not produce the warning line. Their durable
path remains discoverable through `index.json`; a concise non-warning count can
be added to the ordinary success line only if operator testing shows value. An
ordinary successful run with no process degradation should write nothing to
the warning stream even when it publishes quality advisories or normalization
observations.
Create `notarius.run-result.v2` rather than redefining v1. It should expose at
least:
- `warning_group_count`;
- `warning_occurrence_count`; and
- `diagnostic_group_count` for non-warning durable groups.
The receipt should continue to expose validation status, validation summaries,
and rejected-output count independently. Process exit behavior should not
change as part of warning presentation reform.
### Checkpoint Semantics
Store the structured terminal diagnostic groups with accepted checkpoints and
replay them exactly once at their logical stage. Fresh and reused runs should
produce the same public groups and counts. Checkpoint events and debug records,
not diagnostic identity, should disclose whether computation was reused.
### Output Encoder Boundary
Prefer removing `OutputResult.Warnings`. A successful output encoder should
either return the complete logical files or fail. If future encoders genuinely
need to produce durable post-encoding warnings, introduce an explicit
two-phase prepare/finalize contract so those warnings can be included in the
same published collection. Do not retain the current self-inconsistent
one-pass capability.
## Resolution Of Required Design Questions
| Question | Recommendation | Viable alternative and tradeoff |
| --- | --- | --- |
| Explicit severity/disposition or external reason mapping? | Put validated disposition and category in the contract. | A central reason-code registry avoids payload fields but makes new modules depend on a second synchronized policy table and leaves downstream meaning implicit. |
| Keep all detail in `warnings.json` or separate it? | Separate grouped actionable warnings from grouped advisories/observations in `diagnostics.json`. | Keep the flat durable list and aggregate only CLI output; simpler migration, but it preserves the noisy downstream contract and ambiguous count. |
| Who owns aggregation? | Framework runner/coordinator after origin enrichment. | Output module aggregation keeps framework types smaller but duplicates policy across encoders and cannot repair missing validator origin. |
| Stable aggregation key? | Disposition, category, reason code, and full producer origin; exclude chunk/scope/message. | Explicit producer-supplied grouping keys offer flexibility but add another identity that can drift from reason codes. Message-template grouping is brittle and unsafe. |
| Samples and omissions? | Exact occurrence count plus deterministic bounded samples and numeric omitted-sample count. | Omission warning records preserve the current representation but inflate group counts and require prose parsing. |
| `warning_count` semantics? | Version receipt and replace ambiguity with group and occurrence counts. | Keep v1 count as published record length and add optional fields; compatible, but two competing warning counts remain easy to misuse. |
| Which normalization changes remain warnings? | Only exhausted process fallback. Unresolved membership is a data-quality advisory; successful canonicalization, reordering, ID repair, source-ref dedupe, and duplicate consolidation are observations. | Treat unresolved membership or semantic duplicate consolidation as warnings because they affect grounding or cardinality; more conservative, but it violates the process-only warning rule and reports accepted artifact quality as an operational failure. |
| Source-relatedness disposition? | Grouped advisory by default; preserve current approve behavior. | Retain warning disposition or make rejection configurable. Rejection requires production precision evidence; current lexical rules are not strong enough. |
| Checkpoint-loaded warnings? | Present the same logical groups as fresh execution and use checkpoint events for reuse provenance. | Mark groups as replayed; aids forensics but fragments aggregation and makes semantically equivalent runs differ. |
| ADR and schema versions? | Add an ADR and version the run receipt and diagnostic files. | Treat the work as CLI-only presentation and avoid an ADR; insufficient because module contracts, output files, checkpoint payloads, and downstream fields change. |
## Compatibility, Documentation, And ADR Implications
The target alters public and internal contracts enough to require a new ADR.
It should record:
- the distinction among warnings, advisories, observations, rejections, and
errors;
- the invariant that warnings are process-level signals, LLM-judged extraction
quality is never a warning, and ordinary non-degraded success has zero
warnings;
- module semantic ownership versus framework origin/aggregation ownership;
- bounded group and sample semantics;
- fresh/checkpoint equivalence; and
- the output-encoder decision.
Implementation should introduce `notarius.run-result.v2`. The grouped warning
and diagnostic envelopes should each carry their own schema version. Because
the content of `warnings.json` changes incompatibly from a flat array wrapper
to groups, release notes and the published JSON integration contract must call
out the migration. `index.json` gains the diagnostic file path.
Canonical documentation updates belong in:
- `docs/cli.md` for stderr presentation only;
- `docs/operations.md` for operator review and debug workflow;
- `docs/integrations/json-output.md` for warning and diagnostic file schemas;
- `docs/integrations/run-result.md` for v2 fields and compatibility;
- `docs/consumers/subprocess.md` and `docs/consumers/dnd-pipeline.md` for
downstream policy checks;
- `docs/internal/pipeline.md` for promotion, aggregation, retry, and checkpoint
mechanics;
- `docs/internal/modules.md` and `docs/internal/dnd.md` for producer rules and
the D&D classification matrix; and
- `docs/policy/architecture.md` for the durable ownership invariant after the
ADR is accepted and implemented.
No configuration knob is required for the first implementation. A fixed,
well-documented taxonomy is easier to reason about than per-reason display
overrides. Configurable escalation or suppression can be considered only after
production review demonstrates a concrete operator need.
## Test Coverage Assessment
Existing coverage worth preserving includes:
- accepted-attempt and terminal-rejection warning promotion;
- validator failure retry exhaustion and `warn_continue`;
- module semantic retry fallback;
- deterministic warning order under concurrent lane completion;
- chunk-plan invalidation and discarded-cache warning behavior;
- fresh/checkpoint warning equivalence;
- local D&D warning caps and safe dynamic-message quoting;
- JSON warning-file publication; and
- CLI stderr, debug, and receipt counts.
Material gaps are:
- no bound test for NPC- or spell-relatedness warnings;
- no generic warning-field or result-size validation;
- no test for output encoder warnings versus published `warnings.json`;
- no operator-level aggregation or bounded-sample tests;
- no fresh/resume test for grouped counts because groups do not yet exist; and
- no production evaluation of advisory precision.
Tests should protect the semantic relationships: exact occurrence counts,
bounded samples, deterministic group order, actionable-only warning
presentation, and cross-surface equality. They should not assert one exact
warning count for every complete D&D run or treat message wording as a public
API unless the wording itself enforces a security boundary.
## Audit Conclusion
The application is in a good position for warning reform. Its retry,
validation, checkpoint, and concurrency mechanics provide reliable points at
which to attach structured diagnostics. The most valuable change is not to
suppress individual reason codes; it is to replace the semantically flat,
origin-free collection with bounded typed groups and to reserve the word
“warning” for conditions that merit operator attention.
Provider-backed runs would improve prioritization and help tune the D&D
advisories, but they are not necessary to conclude that routine normalization
and heuristic doubt should not dominate stderr or the durable warning
contract. They should be gathered before changing heuristic acceptance policy
or adopting a numerical production warning-volume target.

View File

@@ -1,224 +0,0 @@
# D&D Subprocess Consumer Documentation
## Status
Completed. The target guide is `docs/consumers/dnd-pipeline.md`.
## Purpose
Provide one task-oriented guide for applications that run Notarius as a
subprocess to execute the maintained complete D&D pipeline and consume its
published artifacts. The initial concrete consumer is Narratio, but the guide
must describe the public Notarius workflow rather than depend on Narratio
internals.
The guide should make the safe integration path obvious without duplicating
the CLI, input, receipt, output-bundle, or individual artifact contracts that
already have canonical documentation.
## Current State
The public integration surface is documented accurately but is distributed
across several documents:
- `docs/consumers/subprocess.md` defines the generic subprocess workflow;
- `docs/cli.md` owns commands, flags, stream behavior, and exit statuses;
- `docs/integrations/seriatim.md` owns the accepted transcript input shape;
- `docs/integrations/run-result.md` owns the machine-readable successful-run
receipt;
- `docs/integrations/json-output.md` owns bundle discovery and logical files;
- the D&D integration documents own the individual lane payload contracts;
- `examples/dnd-complete.config.yml` is the maintained complete pipeline.
A consumer can reconstruct the full workflow from those documents, but there
is no D&D-focused guide that connects the maintained example to its input,
invocation, complete artifact inventory, discovery procedure, and downstream
acceptance decisions.
## Target Documentation Set
### Create `docs/consumers/dnd-pipeline.md`
This document should own the end-to-end consumer workflow for the maintained
complete D&D configuration. It should be useful to Narratio and to another
subprocess orchestrator with the same needs.
The guide should contain the following sections.
#### Prerequisites And Deployment Configuration
- Link to `examples/dnd-complete.config.yml` rather than embedding a second
complete configuration.
- Explain that a deployment must provide the configured PromptKit profile and
campaign reference files.
- Recommend absolute paths for a service or orchestrator deployment.
- Call out the path-resolution distinction explicitly: YAML reference paths
are relative to the Notarius configuration file, while
`promptkit.profile_file` is relative to the Notarius process working
directory.
- Recommend validating the selected configuration and `dnd-session` pipeline
before processing sessions.
#### Transcript Input
- State that the complete pipeline consumes a Seriatim JSON document.
- Link to the canonical Seriatim contract for required fields and validation.
- Recommend the caller's final trimmed transcript when the caller maintains
transcript tiers. For Narratio, identify the implemented source as
`narratio.transcript.final_trimmed`, normally stored at
`transcripts/final.trimmed.json`.
- Explain that segment IDs must remain stable because D&D source references
cite those units.
- Explain that Notarius derives its default prompt session from the input
module and exact input bytes and that ordinary callers should not supply
`--session-id`.
#### Subprocess Invocation
- Show one concise invocation using `notarius run dnd-session`, explicit
absolute `--config`, `--input`, and `--output-dir` paths, and `--json`.
- Direct callers to capture stdout and stderr separately, propagate
cancellation, impose an operator-appropriate timeout, and wait for process
completion before parsing stdout.
- State that only exit status zero permits receipt decoding and link to the CLI
contract for the complete exit-status definition.
- Recommend retaining stderr and the invocation context for diagnosis without
logging secrets or transcript content.
#### Receipt And Bundle Discovery
- Require callers to accept only supported run-result schema versions while
tolerating unknown fields allowed by that version.
- Direct callers to obtain the exact run-specific bundle from the receipt's
absolute `output_directory`; they must not scan for the newest run directory
or construct a run ID.
- Require a confinement check when resolving `index_file` beneath the reported
bundle root.
- Direct callers to discover lane payloads by `lane_id` in `index.json`, then
verify descriptor media type and schema identity before decoding them.
- Explain that descriptor paths are untrusted relative paths and require the
same confinement discipline.
#### Complete D&D Artifact Inventory
Include a compact table for the ten lane IDs selected by the maintained
complete configuration:
- `item-registry`;
- `npc-registry`;
- `location-registry`;
- `scene-descriptions`;
- `item-occurrences`;
- `spells`;
- `combat-turns`;
- `npc-occurrences`;
- `location-occurrences`;
- `enemy-events`.
For each row, give a one-line purpose and link to the corresponding canonical
D&D artifact contract. Do not copy its fields or schema rules into the
consumer guide.
Document the four always-published bundle files—`index.json`, `manifest.json`,
`rejected.json`, and `warnings.json`—and the complete example's configured
`chunk-map.json` and `evidence-context.json` pipeline-wide artifacts. Link to
their canonical contracts and distinguish pipeline-wide artifacts from lane
outputs.
The inventory must say that a file is available only when its corresponding
artifact was accepted and published. It must not imply that process success
guarantees every configured lane.
#### Downstream Acceptance And Retention
- Explain that exit status zero can coexist with rejected outputs, warnings,
or absent lane descriptors.
- Require the consumer to define its required lane set explicitly. Recommend
treating all ten lanes as required when the caller claims to consume the
complete D&D workflow, while allowing another consumer to adopt a narrower
documented policy.
- Recommend retaining the receipt, the complete published bundle, and captured
diagnostic streams long enough to support provenance and failure analysis.
- Explain that `evidence-context.json` is a reading excerpt; authoritative
citations remain in lane payloads.
- Treat transcripts, lane artifacts, evidence context, manifests, and logs as
sensitive campaign data.
#### Compatibility Checklist
End with a concise checklist covering process exit, receipt schema, path
confinement, pipeline identity, index decoding, required descriptors,
descriptor schema/media compatibility, warnings and rejections, checksums or
retention, and secure handling. Compatibility should be based on published
receipt and artifact contracts rather than parsing a human version string.
### Update Existing Navigation
- Add a short link from `docs/consumers/subprocess.md` to the D&D-specific
workflow. Keep generic subprocess policy in the existing document.
- Add the guide to the documentation links in `README.md`.
- Extend the subprocess-consumer row in `docs/development.md` so maintainers
working on the D&D workflow are routed to the new guide and the canonical
contracts.
### Verify Canonical Contract Documents
Review the linked integration documents and the complete example while writing
the guide. Correct an integration document only if repository inspection finds
an actual stale contract. Do not move schema definitions, field tables, CLI
flags, or configuration semantics into the new guide.
## Narratio Alignment
The guide may name Narratio as the motivating consumer and identify its current
final-trimmed transcript source. It must not claim that Narratio already has a
Notarius adapter or extraction stage. Until that feature is implemented,
Narratio-specific architecture, configuration, stage behavior, manifest
records, and artifact source IDs belong in Narratio's roadmap.
Once Narratio implements the integration, its own integration documentation
should link to this guide and the durable Notarius contracts instead of
repeating them.
## Validation
Documentation implementation should include:
```sh
go run ./cmd/notarius config validate \
--config examples/dnd-complete.config.yml \
--pipeline dnd-session
go test ./...
```
Also verify all new and changed relative Markdown links, compare the artifact
inventory directly with the maintained complete configuration, and confirm
that commands and path semantics match the CLI and configuration references.
If the repository still has no automated link checker, record that fact and
perform a focused manual link review.
## Acceptance Criteria
- A subprocess integrator can follow one D&D-focused guide from a Seriatim
transcript through safe discovery of every artifact configured by the
complete example.
- The guide makes stdout, stderr, exit-status, receipt, and path-confinement
responsibilities unambiguous.
- The ten configured D&D lanes and both configured pipeline-wide artifacts are
listed and linked to their canonical contracts.
- The guide distinguishes process success from the caller's required-artifact
policy.
- The profile-path and reference-path resolution rules are clearly stated.
- Existing navigation makes the guide discoverable.
- No volatile contract is defined in two places, and no unimplemented Narratio
behavior is presented as current.
## Non-Goals
- Implementing or documenting Narratio's future adapter or stage as current
Notarius behavior.
- Adding a new Notarius command, receipt version, output format, or artifact
schema.
- Duplicating the complete configuration or individual D&D payload schemas in
prose.
- Defining a universal partial-result policy for every Notarius consumer.

View File

@@ -7,197 +7,16 @@ not as committed release dates.
## Near-Term Validation And LLM Reliability ## Near-Term Validation And LLM Reliability
The following work forms one related program but should be promoted into PromptKit now owns structural output repair within one completion. Notarius
separate feature roadmaps and implemented in dependency order. PromptKit owns owns stage candidates, validator chains, semantic rejection policy, bounded
structural output repair within one completion. Notarius owns stage candidates, feedback-aware stage retries, validation provenance, and reusable-state
validator chains, semantic rejection policy, and whether another stage attempt eligibility, and the separation of actionable process warnings from quality
is warranted. diagnostics. The remaining near-term work applies those completed foundations
to domain review and empirical evaluation.
### 1. Upgrade To PromptKit v0.8.0 Near-term reliability work should now be selected from the concrete evaluation
and extension opportunities below. The retry, validation, and subprocess
This item has been promoted to the standalone foundations described above are implemented current behavior.
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md) roadmap. That document owns the
release-by-release compatibility review, adopted features, structured-repair
policy, target integration boundary, acceptance criteria, and settled design
decisions.
### 2. Feedback-Aware Stage Validation Retries
- Model Notarius's corrective stage-retry conversation explicitly after
PromptKit v0.8.0. The first attempt sends the ordinary complete initial
prompt. If application validation rejects the resulting LLM-produced
candidate and another stage attempt is available, reconstruct that complete
initial prompt byte-for-byte and append exactly two messages: an assistant
message containing the defective response and an application-owned user
message detailing every applicable semantic validation error and requesting
one corrected, complete replacement response. This is a freshly constructed
correction request, not continuation of an accumulating conversation.
- Use the configured stage `retries` value as the one outer retry budget for
this loop. `retries: N` continues to mean at most `N` additional complete
chunk, extract, merge, or normalize attempts after the initial attempt,
whether an attempt is needed because of a producer error or semantic
rejection. Do not add a second semantic-correction count. PromptKit's
prompt-level `repair_attempts` budget is independent and internal to each
individual LLM completion, and does not consume or replenish the Notarius
stage budget.
- Extend the framework-managed validation boundary for chunk, extract, merge,
and normalize stages so a rejected LLM-produced candidate and its exact raw
model response remain available to construct the next stage attempt.
Deterministic producers cannot improve by repeating the same inputs; a
rejection from a deterministic stage is therefore terminal under the
configured rejection policy rather than consuming retries mechanically.
- Preserve the original session ID, selected profile, structured-output
contract, prompt inputs, and reusable prompt prefix. Carry only the latest
candidate and latest aggregate feedback; do not build an unbounded retry
conversation. Keep model-facing corrective guidance separate from
operator-facing diagnostics, and apply explicit size, redaction, and debug
disclosure rules to both.
- Run every applicable validator in the configured chain before deciding
whether to retry. Do not short-circuit merely because an earlier validator
rejected the candidate. Aggregate all semantic rejection reason codes and
corrective guidance into the retry message so one retry can address the
whole candidate. A validator is applicable only when its declared target and
prerequisites can be satisfied; record a deterministic skipped diagnostic
rather than invoking a validator on an input it cannot interpret. Initially
execute the chain sequentially in configured order so results, diagnostics,
costs, and feedback ordering remain deterministic; consider validator
concurrency only in response to measured latency.
- Continue running independent applicable validators after one validator
execution failure so the attempt retains as much useful diagnostic
information as practical. Do not present validator operational failures as
defects in the producer candidate and do not include them in corrective
feedback.
- Distinguish three terminal conditions and make their policies configurable
at a coherent pipeline or binding scope:
- **producer structural failure:** PromptKit could not return a usable
structured candidate after its repair budget. Default to `fail_run`; an
allowed alternative may record a terminal stage or lane rejection where
execution can safely continue, but may not accept the invalid output;
- **semantic rejection:** one or more validators completed and rejected the
candidate. Default to `fail_run` after corrective stage retries are
exhausted; allow an explicit alternative that records the existing
rejected-output outcome without advancing that output;
- **validator execution failure:** a validator could not produce a valid
decision because of generation, structural-output, transport, or internal
failure. Default to a genuine warning and an explicitly recorded
`validation_incomplete` or equivalent degraded state while allowing the
candidate to continue; allow strict configuration to fail the run instead.
- An LLM-backed validator uses the same scheduled PromptKit boundary as every
other LLM-backed module. Its own response may use PromptKit's bounded
structural repair. Distinguish its possible output states:
- output rejected by PromptKit's structural contract should consume only the
validator prompt's configured PromptKit repair budget;
- output that is structurally valid but violates a deterministically
checkable validator-result invariant should be classified as a validator
execution failure;
- output that satisfies the complete validator-result contract is the
validator's decision, even though an LLM judgment may remain imperfect.
Automatically judging that judgment would require another semantic
validator and is outside this feature.
If the validator cannot return a contract-valid decision, do not recursively
create another Notarius semantic-validation loop around it. Apply the
configured validator-failure policy. The default warning must identify the
validator and affected stage without exposing sensitive content.
- Separate validator execution retry from producer correction. A transient
validator operational failure must not automatically discard and regenerate
an otherwise usable producer candidate. Any bounded retry of the validator
itself should reuse that same immutable candidate and remain subordinate to
PromptKit and provider retry behavior.
- Preserve attempt-level provenance, cumulative token usage, validator
outcomes, aggregated correction feedback, and terminal policy decisions in
the debug and manifest models without copying raw source material into
ordinary errors or durable summaries.
- Define terminal-outcome precedence. A semantic rejection dominates a
validator execution failure for the same candidate: use the completed
rejections to correct the producer while separately recording incomplete
validation. If a later candidate has no semantic rejection but one validator
still fails, apply the configured validator-failure policy to that candidate.
Never allow a known semantic rejection to become accepted through a
warn-and-continue setting, and never accept a structurally invalid producer
response. Permissive policy may preserve a rejected-output outcome or accept
a structurally valid candidate with explicitly incomplete validation; it may
not relabel known-invalid output as approved.
Before implementation, record the generic validation and retry state machine
in an ADR. The ADR should own the separation between PromptKit repair and
Notarius correction, use of the existing stage-retry budget, reconstruction of
correction conversations, all-applicable-validator aggregation, deterministic
validator ordering, non-recursive validator failure handling, outcome
precedence, default fail-open/fail-closed choices, configurable terminal
policies, and provenance and sensitive-data constraints. A dependency-upgrade
ADR is not needed for PromptKit v0.8.0 itself. Current behavior remains
authoritative until the validation ADR is implemented and the canonical
architecture, configuration, operations, and internal documentation are
updated.
### 3. D&D Combat Scene Semantic Validation
- Add an optional production LLM-backed D&D validator that determines whether
proposed scene boundaries and classifications represent substantive active
combat correctly. Its central quality goal is that active combat is kept in
coherent scenes classified as `combat`, rather than split incorrectly or
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
- Resolve the validator's exact target before implementation. The current
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
`recap`, and `meta` classification. The preferred initial placement is
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
can compare one proposed kind with the corresponding transcript chunk.
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
question that can be answered from the complete transcript and proposed
range map, such as whether one continuous combat was fragmented across
inappropriate scene boundaries. Do not duplicate the same classification
judgment at both stages. Moving classification into chunk-plan annotations
would change the deliberately minimal, annotation-free chunk contract and
requires an explicit architecture review before it is selected.
- Validate both false negatives and false positives: a non-combat kind must not
omit substantive active combat, and a combat kind must be supported by such
combat. Keep the existing deterministic downstream rule that combat-turn
extraction runs only for an exact `combat` scene classification; semantic
review improves the upstream classification but does not replace that gate.
- Run the semantic validator through PromptKit, use a minimal required-field
structured response schema, and let PromptKit repair structural validator
output within its bounded budget. A contract-invalid final validator response
is a validator execution failure, not a semantic rejection and not a reason
to recursively validate the validator.
- Evaluate the prompt and decision policy against a small human-reviewed set
containing combat setup, active turns, interruptions, multi-phase encounters,
brief rules discussion, aftermath, recalled combat, and false-positive
hostile dialogue. Measure false acceptance, false rejection, retry success,
added calls, latency, and token cost before placing it in the production
default chain.
- An ADR is not required if classification remains owned by
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
Create or supersede an ADR if the work transfers scene classification into
the chunker or otherwise changes stage ownership or the durable chunk-plan
contract.
### 4. Warning Signal And Presentation Reform
- Audit every warning producer and representative successful runs. Ordinary
success producing dozens of warnings is a failed operator experience: the
volume obscures actionable problems and trains operators to ignore the
warning channel.
- Define a small warning taxonomy that distinguishes actionable degradation,
incomplete validation, lossy fallback, and data-quality risk from routine
normalization observations or informational diagnostics. Preserve detailed
traceability in debug or manifest data without promoting every observation
to a top-level CLI warning.
- Consider stable deduplication and aggregation by scope and reason code,
bounded samples plus omitted counts, and a concise CLI summary with a path to
detailed diagnostics. Do not suppress genuine validator execution failures
merely to reduce the count.
- Decide which warnings affect process status, rejection summaries, durable run
receipts, or only debug output. Ensure warning ordering and aggregation are
deterministic across concurrent execution.
- Establish a representative warning-volume acceptance target and human review
workflow before changing individual producers piecemeal. The intended result
is not zero warnings; it is a small set in which every surfaced warning merits
operator attention.
- This work does not require an ADR unless it changes validation acceptance,
failure, or durable contract semantics. CLI presentation and diagnostic
taxonomy otherwise belong in a feature roadmap followed by updates to their
canonical configuration, operations, integration, and internal documents.
## Near-Term D&D Pipeline ## Near-Term D&D Pipeline

View File

@@ -1,782 +0,0 @@
# PromptKit v0.8.0 Upgrade Implementation Plan
## Purpose
Implement the target state defined by the
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md): adopt the useful PromptKit
v0.6.0, v0.7.0, and v0.8.0 changes; enable one bounded structural correction
by default; expose pipeline and binding overrides; preserve safe provider
diagnostics; and keep PromptKit behind Notarius's transport-neutral LLM
boundary.
This plan is ordered. Each numbered stage is one implementation prompt for a
gpt-5.6-terra coding agent. Complete and validate one stage before beginning
the next. Read `docs/development.md` and every policy under `docs/policy/` at
the start of each stage, inspect the current code and tests named by that
stage, preserve unrelated worktree changes, and update current-behavior
documentation in the same stage as the behavior it describes.
Do not retire this plan or `promptkit-v0.8.md` during implementation. Keep both
until the completed work has passed a separate review. Do not implement the
future Notarius semantic-validation retry loop, D&D combat-scene validator, or
warning redesign as part of this plan.
## Decisions Fixed For Implementation
- Pin `gitea.maximumdirect.net/eric/promptkit` v0.8.0 directly, with no
`replace`, workspace dependency, or vendored source.
- Every maintained eligible production prompt defaults to exactly one
PromptKit structural repair attempt.
- Add the exact configuration key
`structured_output_repair_attempts` at pipeline scope and on LLM-backed
module and validator bindings.
- Effective precedence is binding value, then pipeline value, then the prompt's
declared `repair_attempts` value. Omission inherits; explicit zero disables
structural repair at that scope.
- Accepted values are integers from zero through three. Explicit null and
non-integer values are invalid. An explicit binding value on a deterministic
module or validator is invalid. A pipeline value is applied only to selected
LLM-backed bindings and does not make deterministic bindings invalid.
- Keep file configuration version 4. This is an additive pre-release field and
does not require parallel versioned behavior.
- Use `StructuredOutputRepairAttempts *int` for presence-aware internal Go
fields. Clone pointers at every ownership boundary.
- A configured override never replaces schema identity, output format, or
validation mode. The PromptKit adapter calls `InspectPrompt`, copies the
complete normalized prompt-owned output contract, changes only
`RepairAttempts`, and supplies the complete replacement on `RunRequest`.
Do not add an inspection cache initially.
- PromptKit repair is internal to one `CompleteStructured` call and does not
consume or replenish a binding's existing `retries` budget.
- Add `RepairAttempts int` to Notarius's structured-completion response. It is
the actual corrective-call count reported by PromptKit; token usage remains
PromptKit's cumulative usage and must not be summed again.
- A valid repaired response is successful and produces no warning solely
because repair occurred. Exhausted structural validation maps to
`ErrInvalidStructuredOutput` with the final candidate and debug material
retained.
- Add an application-owned generation-error sentinel and typed status-bearing
error. PromptKit error types must not cross `internal/framework/llm`.
- HTTP status may appear in the application-owned generation error. Provider
code, type, and message are excluded from ordinary errors, warnings,
manifests, cache, and checkpoint identity; they may appear only in an
explicitly requested debug trace after Notarius redaction.
- Profile inheritance is owned entirely by PromptKit. Notarius passes sources
through, inspects and records the resolved target, and does not parse or merge
`base_profile` itself.
- PromptKit's built-in `rakestrawhome` backend and
`rakestrawhome-gemma-4-31b` profile are available generically. Notarius does
not register, shadow, or select them by default.
- Missing optional credential environment values are allowed to reach the
provider without `Authorization`; Notarius does not recreate v0.5.0's local
failure or add provider-specific authentication logic.
- No dependency-upgrade ADR is required. Update architecture only with the
durable ownership distinction between PromptKit structural repair and
Notarius stage/semantic validation policy.
## Stage 1: Upgrade The Dependency And Establish A Clean v0.8.0 Baseline ✅
### Goal
Move the repository to PromptKit v0.8.0, resolve source-compatibility issues,
and establish a passing baseline before adopting new behavior.
### Implementation
1. Re-read the upstream v0.6.0, v0.7.0, and v0.8.0 release guides and the
v0.8.0 package consumer and format documentation. Treat the pinned v0.8.0
tag, not the sibling checkout's moving branch, as authoritative.
2. Update `go.mod` and `go.sum` to PromptKit v0.8.0 and run `go mod tidy` with
`GOWORK=off`.
3. Compile before making compatibility edits. Correct only actual source or
behavior incompatibilities. In particular:
- convert any positional `promptkit.Profile` or
`promptkit.OpenAICompatibleProfileConfig` literals to keyed literals;
- confirm Notarius does not register the newly reserved `rakestrawhome`
backend ID; and
- preserve `PrepareExecution`/`Details`/`RunPrepared` snapshot ownership,
`Discard`, session forwarding, reasoning override, profile preflight,
and capacity adaptation.
4. Change `promptKitBuiltinProfileCatalogID` in
`internal/framework/llm/promptkit_profile_fingerprint.go` from the v0.5.0
catalog marker to an opaque v0.8.0 marker. Do not hash PromptKit internal
files or include catalog content in manifests.
5. Update `docs/integrations/pkg-promptkit.md` to pin and link v0.8.0 and to
state that this stage still leaves the production prompt-declared repair
budget at its current value. Do not document later configuration or default
behavior before it exists.
6. Update only those existing tests whose public PromptKit types or stable
v0.8.0 behavior genuinely changed. Do not rewrite tests merely to match
upstream diagnostic wording.
### Tests And Validation
```sh
GOWORK=off go mod tidy -diff
GOWORK=off go test ./internal/framework/llm ./internal/cli
GOWORK=off go test ./...
GOWORK=off go vet ./...
GOWORK=off go build ./cmd/notarius
git diff --check
```
### Acceptance Criteria
- `go list -m gitea.maximumdirect.net/eric/promptkit` reports v0.8.0.
- There is no PromptKit `replace`, active Go workspace dependency, or vendor
tree.
- The adapter still uses one frozen prepared execution and all existing LLM
tests pass.
- Checkpoint profile identity includes the v0.8.0 built-in catalog marker.
- Current integration documentation pins v0.8.0 without claiming that
later stages are already active.
- The full ordinary test suite, vet, and command build pass.
## Stage 2: Verify v0.6.0 Compatibility And Hardening ✅
### Goal
Audit Notarius's assets and boundary values against PromptKit v0.6.0's stricter
source, path, endpoint, JSON, and cancellation contracts, fixing only concrete
incompatibilities.
### Implementation
1. Inspect `internal/framework/llm/asset_registry.go`, prompt/profile source
composition, all registered asset roots, the conventional local backend,
and their focused tests.
2. Exercise every production asset registry through PromptKit engine
construction and the existing production composition tests. Confirm that:
- YAML IDs and versions, not filenames, select definitions;
- every `content_file` path is exact, relative, contained, and points to a
regular embedded file;
- every schema and JSON asset is one complete JSON value;
- every current output contract is valid under v0.8.0; and
- unrelated malformed definitions do not create a second Notarius identity
or fallback mechanism.
3. Review local endpoint parsing and validation. Retain a narrower Notarius
rule only if it has independent application value; otherwise rely on
PromptKit's absolute HTTP/HTTPS URL contract. Never accept a value that the
adapter will later reject.
4. Review conversion of Notarius variables, inputs, profile extras, and debug
values at the adapter boundary for PromptKit's bounded JSON-compatible-value
rules. Do not add a second generic JSON walker or duplicate upstream numeric
limits.
5. Verify cancellation and deadline identity through existing adapter tests.
Add or refine one focused regression only if Notarius currently destroys an
`errors.Is`-relevant context or transport error that the application owns.
6. Do not add a cross-operation schema cache, artifact cache, provider-body
reader, or duplicate JSON framing validation; v0.6.0 owns those mechanisms.
### Tests And Validation
Run the focused asset, profile-source, and adapter packages, then the ordinary
and race-enabled suites:
```sh
GOWORK=off go test ./internal/framework/llm ./internal/cli
GOWORK=off go test ./...
GOWORK=off go test -race ./...
git diff --check
```
Tests must remain offline and should validate Notarius's assembled boundary,
not reproduce PromptKit's internal path, JSON-depth, or response-size matrices.
### Acceptance Criteria
- Every maintained embedded prompt, schema, and fallback profile can be loaded
through the assembled v0.8.0 engine.
- Current local endpoint and JSON-compatible values either satisfy the stricter
upstream contract or fail during preparation with safe diagnostics.
- No duplicate PromptKit-owned cache, JSON, or response-bound mechanism is
introduced.
- Cancellation and deadline behavior remains discoverable at the Notarius
boundary.
- Ordinary and race-enabled tests pass.
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ✅
### Goal
Make the useful PromptKit v0.7.0 profile and backend behavior work through
Notarius's existing generic profile boundary without adding provider-specific
composition logic.
### Implementation
1. Inspect `promptkit_profiles.go`, `asset_registry.go`, profile fingerprinting,
CLI profile preflight, profile provenance recording, and their tests before
editing.
2. Add an offline integration test using a temporary operator profile source
whose leaf uses `base_profile`. Prove that:
- preflight reports the leaf ID;
- the effective backend, model, reasoning, and other inherited values match
the resolved PromptKit target;
- execution uses the same resolved target as inspection; and
- a missing parent or cycle fails before provider generation with a safe
profile-load diagnostic.
Do not duplicate PromptKit's entire field-by-field merge test matrix.
3. Add a checkpoint-safety test showing that changing a parent definition in
an operator profile directory changes Notarius's profile-source fingerprint
while profile content and paths remain absent from the fingerprint value.
Retain the v0.8.0 catalog marker as coverage for built-in-parent changes.
4. Verify `rakestrawhome-gemma-4-31b` through the ordinary profile inspector.
Assert its selected backend reaches Notarius's application-owned inspection
and provenance fields. Use a fake PromptKit client or transport if execution
coverage is needed; never contact the live service or require credentials.
5. Verify that Notarius registers no `rakestrawhome` override and that the
existing `local` registration remains independent.
6. Add one `httptest`-backed adapter integration test for a filesystem profile
with a missing optional `api_key_env`. The request must reach the test server
without an `Authorization` header. Add a focused in-memory PromptKit profile
test for `APIKeyRequired` only if needed to prove Notarius preserves upstream
preflight behavior; do not expose a new operator profile API.
7. Keep `assets/dnd/profiles/dnd-extraction.yaml` standalone and unchanged. No
matching v0.8.0 built-in profile owns its `openai/gpt-5.6-luna` target.
8. Update the current profile-source, deployment, and pinned-integration
sections in `docs/config.md`, `docs/operations.md`,
`docs/internal/llm.md`, and `docs/integrations/pkg-promptkit.md`. Link to the
pinned PromptKit format rules for inheritance. Explain that filesystem
profiles cannot express PromptKit's in-memory `APIKeyRequired` field and
that an optional missing credential may result in a provider 401/403.
### Tests And Validation
```sh
GOWORK=off go test ./internal/framework/llm ./internal/core/config ./internal/cli
GOWORK=off go test ./...
GOWORK=off go test -race ./internal/framework/llm ./internal/cli
git diff --check
```
### Acceptance Criteria
- Inherited operator profiles resolve identically during preflight and
execution, with the leaf ID and effective target kept distinct.
- Parent changes invalidate checkpoint reuse without leaking profile content or
paths.
- Rakestrawhome is available through generic PromptKit profile handling and is
not selected by default or registered by Notarius.
- Missing optional credentials omit authorization and reach the controlled
test provider; explicitly required credentials retain upstream behavior.
- Current documentation accurately describes the implemented profile and
credential behavior without duplicating PromptKit's merge algorithm.
## Stage 4: Adapt Structured Generation Errors Safely ✅
### Goal
Use PromptKit v0.7.0's structured generation errors for stable status
classification and debug-only provider diagnostics without leaking PromptKit
types or sensitive provider text.
### Implementation
1. In `internal/framework/contracts`, add:
- `ErrLLMGeneration` as the provider-neutral generation-failure sentinel;
- an application-owned `LLMGenerationError` with private status and safe
diagnostic fields, `Error`, `Unwrap`, and `StatusCode` methods; and
- a constructor that accepts a nonnegative status and an already-redacted
diagnostic. Status zero means no HTTP status was available.
Ordinary callers may inspect status with `errors.As` and category with
`errors.Is`, but cannot obtain provider code, type, or message from the
error.
2. Add an application-owned `LLMDebugProviderError` with `status_code`, `code`,
`type`, and `message` fields, referenced optionally from
`LLMDebugResponse`. This is debug material, not a manifest or durable public
artifact contract.
3. In `PromptKitClient.CompleteStructured`, preserve precedence in this order:
caller context cancellation/deadline, PromptKit capacity error, structured
PromptKit generation error, then other PromptKit generation failures.
Map every generation failure to `ErrLLMGeneration`; map
`*promptkit.GenerationError` to `LLMGenerationError` with its status.
Never wrap or return the PromptKit error value itself.
4. Keep the ordinary diagnostic limited to PromptKit's safe default error
formatting after bearer and known-credential redaction. Do not append
`ProviderCode`, `ProviderType`, or `ProviderMessage` to it.
5. For an explicitly requested debug path, preserve prepared prompt details and
attach the PromptKit provider code, type, and message after:
- reading only the selected prepared target's `APIKeyEnv`, if any, to obtain
the exact known credential solely for redaction;
- applying `RedactSecrets` and the existing bearer/key-pattern redaction;
- retaining PromptKit's already-normalized bounds; and
- discarding the credential value immediately rather than storing it.
Do not scan unrelated environment variables.
6. Return prompt/debug material alongside the error so the existing debug LLM
wrapper can persist it only when debug recording is enabled. Confirm that
provider fields do not appear in ordinary error text, warnings, manifests,
cache, checkpoint data, or a run without debug output.
7. Refactor error mapping into small helpers if needed to keep
`CompleteStructured` readable; do not create provider-specific policy in
modules or the pipeline runner.
8. Update the error and observability sections of `docs/internal/llm.md` and
`docs/integrations/pkg-promptkit.md`. Keep operator disclosure rules in
`docs/operations.md` concise and link to the internal boundary where useful.
### Tests And Validation
- Use `httptest.Server` to return representative structured 400 and 503
responses. Assert `errors.Is(ErrLLMGeneration)`, `errors.As` to the
application-owned type, and the exact status without asserting complete
human wording.
- Include a provider message containing the selected test credential and a
bearer-shaped value. Verify both are absent from the ordinary error and
debug artifact, while a non-sensitive marker appears only in the requested
debug trace.
- Retain existing capacity and context tests to prove their more specific
classifications still win.
```sh
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline ./internal/cli
GOWORK=off go test ./...
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
git diff --check
```
### Acceptance Criteria
- PromptKit generation errors never escape the adapter error chain.
- All generation failures match `ErrLLMGeneration`; structured non-success
responses expose only application-owned HTTP status to ordinary callers.
- Provider code, type, and message are available only in an explicitly
requested, redacted debug trace.
- Capacity and context classifications remain unchanged and more specific.
- Security tests prove selected credentials and bearer tokens are not leaked.
## Stage 5: Add Adapter-Level Structured Repair Support ✅
### Goal
Teach the transport-neutral completion boundary and PromptKit adapter to apply
an optional repair override and report actual repair behavior, without yet
exposing the setting in pipeline configuration.
### Implementation
1. Add `StructuredOutputRepairAttempts *int` to
`contracts.StructuredCompletionRequest`. Copy the pointed-to value wherever
requests are cloned or retained.
2. Add `RepairAttempts int` to `contracts.StructuredCompletionResponse`. It is
the actual number of corrective generation calls, not the configured budget
and not the number of total candidates.
3. Validate a non-nil request value as zero through three at the adapter
boundary so programmatic callers cannot bypass later file/config validation.
4. When the request value is nil, leave `promptkit.RunRequest.Validation` nil
so the prompt's complete contract remains authoritative.
5. When the value is non-nil:
- call `Engine.InspectPrompt(ctx, promptID, promptVersion)`;
- copy `PromptInspection.OutputContract` by value;
- replace only `RepairAttempts`;
- pass the complete copied contract as `RunRequest.Validation`; and
- prepare and execute exactly as before.
Do not infer or hard-code schema paths, validation modes, or formats. Do not
cache inspection in this stage.
6. Map `result.Validation.RepairAttempts` to the response and leave
`result.Usage` cumulative values unchanged. The existing debug validation
object and prepared output contract should show actual and configured values
respectively.
7. Preserve result semantics:
- valid initial and repaired candidates decode normally;
- repair exhaustion returns the final raw candidate/debug material with an
error matching `ErrInvalidStructuredOutput`;
- explicit empty or whitespace-only content follows PromptKit validation;
- missing/null/non-string content remains a generation/provider failure;
- corrective-call generation errors use Stage 4's application-owned mapping;
and
- context cancellation wins at every error boundary.
8. Keep `CompleteStructured` and its helpers provider neutral outside this
adapter package. Do not expose PromptKit validation or inspection types.
9. Update only the adapter-owned repair behavior in `docs/internal/llm.md` and
`docs/integrations/pkg-promptkit.md`. State that public pipeline configuration
and the production default are added by later stages of this plan.
### Tests And Validation
Add adapter-level behavioral tests using a deterministic fake PromptKit LLM:
- nil override uses the prompt declaration;
- explicit zero overrides a positive prompt declaration without dropping its
JSON Schema contract;
- explicit one turns an invalid first candidate followed by a valid candidate
into one successful response with the final raw bytes, actual repair count
one, and cumulative usage;
- repair exhaustion returns the final candidate and validation diagnostics as
`ErrInvalidStructuredOutput`;
- explicit empty content is eligible for repair;
- a corrective generation failure maps through Stage 4; and
- invalid direct values below zero or above three fail before provider work.
Do not assert PromptKit's exact assistant/user correction prose or copy its
full internal repair matrix.
```sh
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
GOWORK=off go test ./...
git diff --check
```
### Acceptance Criteria
- The adapter changes only repair count when applying a request override.
- Nil and explicit zero remain distinct.
- Repaired success returns final raw output, cumulative usage, and actual count
without a warning.
- Exhaustion, empty content, corrective generation failure, and cancellation
match the target semantics.
- No PromptKit type crosses the LLM package boundary.
## Stage 6: Propagate Repair Policy Through Framework Requests ✅
### Goal
Carry an optional effective repair budget from each resolved stage or validator
binding to its module request without changing public file configuration yet.
### Implementation
1. Add `StructuredOutputRepairAttempts *int` alongside `LLMProfile` to every
stage request that can belong to an LLM-backed binding:
- `ParseRequest`;
- `ChunkRequest`;
- `TypedExtractionRequest`;
- `TypedMergeRequest`;
- `TypedNormalizeRequest`;
- `OutputRequest`;
- `TypedValidationRequest`;
- `ChunkValidationRequest`; and
- `SerializedValidationRequest`.
2. Add the same optional field to the erased/internal request carriers used by
registry builders, preparation, runner stage attempts, validator targets,
retry closures, and debug wrappers. Copy pointer values; never share a
mutable pointer owned by configuration.
3. At every runner stage invocation, obtain the value from the exact resolved
producer binding. At every validator invocation, obtain it from that exact
resolved validator binding. Do not use the producer's value for a validator
or vice versa.
4. Ensure all retry attempts for the same binding receive the same effective
structural-repair value. Do not decrement it in Notarius; PromptKit owns the
inner budget independently on each `CompleteStructured` call.
5. Extend `semanticreconcile.Request` with the optional field and carry it into
each generic reconciliation completion. A batched reconciliation may make
several completion calls; each call receives the same effective budget.
6. Update registry erasure/adaptation code for typed merge, normalize, and
validation requests so no field is lost. Preserve input/output support even
though current production input and output modules are deterministic.
7. Add focused framework tests for one chunk producer, one extraction
producer, one normalizer, and one LLM-backed validator. Verify exact pointer
value propagation and separation between producer and validator settings.
Do not add repetitive tests for every generic adapter.
### Tests And Validation
```sh
GOWORK=off go test ./internal/framework/contracts ./internal/framework/pipeline ./internal/framework/semanticreconcile
GOWORK=off go test -race ./internal/framework/pipeline ./internal/framework/semanticreconcile
GOWORK=off go test ./...
git diff --check
```
### Acceptance Criteria
- Every stage and validator request can carry a detached optional repair value.
- The runner sources the value from the exact resolved binding.
- Producer and validator values cannot overwrite one another.
- Stage retries reuse but do not mutate or consume the inner repair budget.
- Semantic reconciliation forwards the budget to every one of its completion
calls.
- Existing behavior remains unchanged while all values are nil.
## Stage 7: Forward Repair Policy From Every LLM-Backed Module ✅
### Goal
Complete the internal end-to-end path by having every production LLM-backed
module forward its stage request value to `CompleteStructured`.
### Implementation
1. Inventory every production `CompleteStructured` call with code search before
editing. The expected current owners include:
- `dnd/scenes` chunking;
- the combat-turn, enemy-event, item-occurrence, item-registry,
location-occurrence, location-registry, NPC-occurrence, NPC-registry,
scene-description, and spell extractors; and
- generic semantic reconciliation used by the item, location, and NPC
registry normalizers.
Reconcile this list with the actual repository; do not omit a newly added
production caller merely because it is not named here.
2. In each direct caller, set
`StructuredCompletionRequest.StructuredOutputRepairAttempts` from the
corresponding stage request. Clone the pointer or use a small shared helper
if that reduces repeated ownership mistakes without moving domain logic.
3. Ensure D&D registry normalizers pass their typed normalize request value into
`semanticreconcile.Request`, and that the generic engine forwards it as
established in Stage 6.
4. Update existing module prompt-mapping tests that already inspect a captured
structured-completion request to assert the new field. Do not create a new
one-test-per-module suite solely to memorialize field plumbing; rely on the
existing request-contract tests plus a final complete call-site audit.
5. Search again after editing for production `CompleteStructured` calls and
verify each either forwards the field or documents why it cannot receive a
pipeline binding. Test-only fakes need only preserve the field when their
contract test depends on it.
6. Do not set a module-specific fallback value. Nil must reach the adapter so
the prompt declaration remains authoritative.
### Tests And Validation
Run focused D&D and semantic-reconciliation packages, then the full suite:
```sh
GOWORK=off go test ./internal/modules/dnd/... ./internal/framework/semanticreconcile
GOWORK=off go test -race ./internal/modules/dnd/... ./internal/framework/semanticreconcile
GOWORK=off go test ./...
git diff --check
```
### Acceptance Criteria
- Every production LLM-backed completion receives the exact stage or validator
repair value.
- No module invents a default or imports PromptKit.
- Registry normalizers preserve the value through semantic reconciliation.
- Existing request-contract tests remain concise and pass.
- A final call-site audit finds no silent production omission.
## Stage 8: Add The Public Repair Configuration Contract ✅
### Goal
Add presence-aware pipeline and binding configuration for
`structured_output_repair_attempts` without yet changing runtime resolution.
### Implementation
1. Add `StructuredOutputRepairAttempts *int` to
`pipeline.PipelineProfile` and `pipeline.ModuleBinding`, using
`json:"structured_output_repair_attempts,omitempty"`.
2. Add presence-aware YAML support at pipeline and object-binding scope:
- exact key `structured_output_repair_attempts`;
- integer values zero through three;
- explicit null, non-integer, and out-of-range values rejected with scoped
diagnostics; and
- scalar shorthand bindings continue to omit the binding override.
Preserve file configuration version 4.
3. Update every configuration clone, conversion, redaction, summary, and JSON
round-trip carrier. Copy pointers by value into newly allocated storage so
parsed, configured, and redacted values do not alias.
4. Preserve omission versus explicit zero through YAML parsing, profile
inheritance, module-binding object form, and JSON round trips. Keep scalar
shorthand bindings equivalent to omission.
5. Do not add a top-level `promptkit.repair_attempts` setting or CLI override.
6. Add concise parser and ownership tests. Defer execution-class checks,
effective precedence, resolved digests, and runtime forwarding to Stage 9,
where module metadata is available.
### Tests And Validation
At the parser/config boundary, test omitted, explicit zero, positive bounds,
negative, above-three, null, non-integer, scalar shorthand, cloning, redaction,
and JSON round-trip behavior. Use relational boundary tests for the allowed
range and avoid duplicating the same cases at every layer.
```sh
GOWORK=off go test ./internal/core/config ./internal/cli
GOWORK=off go test -race ./internal/core/config
GOWORK=off go test ./...
go run ./cmd/notarius config validate \
--config examples/dnd-minimal.config.yml \
--pipeline dnd-session
go run ./cmd/notarius config validate \
--config examples/dnd-complete.config.yml \
--pipeline dnd-session
git diff --check
```
### Acceptance Criteria
- The exact public field parses at pipeline and object-binding scope with the
fixed range.
- Nil and explicit zero remain distinguishable through parsing, cloning,
inheritance, redaction, summaries, and round trips.
- Both maintained configurations remain valid without requiring the new field.
- No runtime or prompt default has changed prematurely.
## Stage 9: Resolve And Apply Repair Configuration ✅
### Goal
Resolve the public field against module execution classes, incorporate the
effective value into pipeline identity, and connect it to the request plumbing
completed in Stages 6 and 7.
### Implementation
1. During resolution, compute the effective value for every selected binding:
- explicit binding value wins;
- otherwise an explicit pipeline value applies to an LLM-backed binding;
- otherwise leave nil for prompt-owned policy.
Apply the pipeline value to LLM-backed validators as well as producers.
2. Reject an explicit binding value on a deterministic module or deterministic
validator using the same execution-class knowledge used for `llm_profile`.
Do not reject a pipeline-level value merely because a selected pipeline also
contains deterministic bindings; simply do not apply it to those bindings.
3. Clone every resolved pointer so the parsed pipeline, resolved profile,
redacted summaries, and runner requests have distinct ownership.
4. Include the effective field in resolved pipeline JSON and digest input. A
change between nil, zero, and a positive value must change the resolved
digest when it changes an LLM-backed selected binding. Unselected lanes must
retain the repository's existing digest and selection semantics.
5. Pass the resolved value into the Stage 6 request field for every selected
input, chunk, extract, merge, normalize, output, and validator binding.
6. Update `docs/config.md` as the canonical field, range, and precedence
contract; `docs/internal/pipeline.md` as the resolution owner; and
`docs/operations.md` for the distinction from binding `retries`. The
prompt-owned production default remains unchanged until Stage 10.
7. Add focused resolution and runner tests. Cover representative execution
classes rather than repeating the same assertion for every module type.
### Tests And Validation
Test:
- binding over pipeline over nil precedence;
- inheritance into each selected LLM-backed stage and validator;
- no inheritance into deterministic bindings;
- explicit deterministic-binding rejection;
- detached pointers;
- runner forwarding for representative producer and validator bindings; and
- digest changes for execution-relevant nil, zero, and positive changes.
```sh
GOWORK=off go test ./internal/framework/pipeline ./internal/cli
GOWORK=off go test -race ./internal/framework/pipeline
GOWORK=off go test ./...
go run ./cmd/notarius config validate \
--config examples/dnd-minimal.config.yml \
--pipeline dnd-session
go run ./cmd/notarius config validate \
--config examples/dnd-complete.config.yml \
--pipeline dnd-session
git diff --check
```
### Acceptance Criteria
- The exact public field has the fixed binding-over-pipeline-over-prompt
precedence for every selected LLM-backed producer and validator.
- Deterministic binding misuse fails during resolution before execution, while
a pipeline value coexists with deterministic bindings.
- Nil and explicit zero remain distinguishable through resolution, runtime,
summaries, and digests.
- A policy change invalidates checkpoint identity when it changes an effective
selected binding.
- Current configuration, pipeline, and operations documentation matches the
implemented behavior.
## Stage 10: Enable The Default, Finish Documentation, And Verify The Feature ✅
### Goal
Set the accepted production default of one repair, reconcile all canonical
documentation, and run the full repository verification pass.
### Implementation
1. Change `repair_attempts: 0` to `repair_attempts: 1` in every maintained
production prompt manifest that produces structured output, including the
generic semantic-reconciliation prompt and every D&D chunk, extraction, and
registry-normalization prompt. Do not mechanically change unrelated test
fixtures whose purpose is to exercise zero.
2. Inspect every production prompt output contract after the edit. Confirm that
each positive budget uses `basic`, `json`, or `json_schema`, remains no
greater than three, and retains its existing format and schema path.
3. Add or refine the smallest durable assembled-assets test that proves the
production engine can prepare the maintained prompts with the activated
contracts. Do not add a brittle test that asserts an exact prompt count,
file count, message prose, correction text, or asset length. The public
default may be tested at one canonical assembled boundary because its
literal value is an operational contract.
4. Confirm a successful repair does not create a warning and that exhausted
repair remains `ErrInvalidStructuredOutput`. Verify the debug prompt records
the configured contract, the debug response records actual repair count,
and cumulative usage is not double-counted.
5. Confirm scheduling behavior with one focused test or existing coverage: the
Notarius scheduled client admits one logical `CompleteStructured` operation
while PromptKit may make serial corrective provider calls inside it. Do not
attempt to reacquire a Notarius permit from inside PromptKit or add a second
scheduler.
6. Reconcile current-state documentation:
- `docs/integrations/pkg-promptkit.md` owns the pinned upstream boundary;
- `docs/config.md` owns field names, range, default, and precedence;
- `docs/operations.md` owns latency/cost, optional credentials, concurrency,
timeout, and the upper-bound formula;
- `docs/internal/llm.md` owns inspection-based contract replacement,
cumulative usage, actual repair count, generation errors, and debug data;
- `docs/internal/pipeline.md` owns effective policy propagation and the
separation from stage retries; and
- `docs/policy/architecture.md` adds only the durable rule that PromptKit
owns deterministic structural repair inside one completion while Notarius
owns stage attempts and semantic validation.
7. Remove current-behavior claims that PromptKit is v0.5.0, that every
production repair budget is zero, or that PromptKit is always single-pass.
Do not alter historical release notes or archived roadmaps.
8. Keep the maintained minimal and complete examples secret-free and valid.
They may omit the new field to demonstrate the default; do not add a
redundant complete profile or Rakestrawhome example merely to exercise an
upstream catalog entry.
9. Review `docs/roadmap/future.md` only for consistency. Leave the future
feedback-aware stage retry, combat-scene validator, and warning-reform work
unimplemented and clearly separate.
### Tests And Validation
Run focused tests first, then all repository checks:
```sh
GOWORK=off go test ./internal/framework/llm ./internal/framework/pipeline ./internal/framework/semanticreconcile ./internal/modules/dnd/...
GOWORK=off go test ./...
GOWORK=off go test -race ./...
GOWORK=off go vet ./...
GOWORK=off go build ./cmd/notarius
GOWORK=off go mod tidy -diff
go run ./cmd/notarius config validate \
--config examples/dnd-minimal.config.yml \
--pipeline dnd-session
go run ./cmd/notarius config validate \
--config examples/dnd-complete.config.yml \
--pipeline dnd-session
git diff --check
```
Also perform focused repository searches that exclude `docs/roadmap/archive/`
and historical release notes:
- no active v0.5.0 PromptKit pins or links remain;
- no maintained production prompt still declares `repair_attempts: 0`;
- every production `CompleteStructured` caller forwards the repair field; and
- no provider code, type, or message is added to ordinary errors, warnings,
manifests, cache, or checkpoint schemas.
If the repository's source-release checker is available and the ordinary
checks above pass, run `./scripts/check-release-source.sh v0.0.0` as the final
integrated validation. It must not create a tag, release note, or repository
artifact.
### Acceptance Criteria
- Every maintained structured prompt defaults to one corrective call and can
be overridden to zero through three at pipeline or binding scope.
- A real assembled Notarius completion follows the PromptKit v0.8.0 repair
contract without changing prompt schema identity or cacheable prefix.
- Actual repair count, cumulative usage, error classification, debug-only
provider diagnostics, scheduling, and checkpoint identity match the feature
roadmap.
- Profile inheritance, Rakestrawhome availability, optional credentials, and
v0.6.0 hardening remain covered and documented.
- All canonical documentation describes implemented v0.8.0 behavior in its
assigned home and leaves future semantic validation work in the roadmap.
- Maintained examples validate, all ordinary/race/vet/build/module checks pass,
and the worktree contains no generated or sensitive artifacts.

View File

@@ -1,520 +0,0 @@
# PromptKit v0.8.0 Upgrade
## Status
Proposed.
## Purpose
Upgrade Notarius from PromptKit v0.5.0 to v0.8.0 and deliberately adopt the
useful correctness, profile-composition, provider-diagnostic, backend, and
structured-output-repair capabilities introduced in PromptKit v0.6.0, v0.7.0,
and v0.8.0.
The upgrade should improve structured-output reliability without confusing
PromptKit's bounded deterministic repair with Notarius's existing stage retry
budget or the future feedback-aware semantic-validation loop. PromptKit types
and provider behavior must remain behind Notarius's transport-neutral LLM
boundary.
## Current State
Notarius currently pins PromptKit v0.5.0. Its production adapter prepares one
frozen execution, records credential-redacted details, and runs that same
prepared value. It maps PromptKit capacity failures to an application-owned
error, maps failed structured validation to `ErrInvalidStructuredOutput`, and
returns PromptKit's raw validated bytes and usage metadata.
Every maintained production prompt uses JSON Schema validation and currently
declares `repair_attempts: 0`. Notarius stage bindings separately expose
`retries`, which reruns a complete stage operation after an error or rejected
candidate. The two mechanisms have different ownership and must remain
independent.
Notarius also maintains:
- embedded prompt, schema, and fallback-profile filesystems;
- operator profile-file and profile-directory sources;
- one optional conventional `local` backend registration;
- explicit profile preflight through PromptKit inspection;
- one application-wide scheduled LLM client around the PromptKit adapter;
- PromptKit profile-source fingerprints for checkpoint safety; and
- redacted debug and manifest provenance at application-owned boundaries.
The upgrade must preserve those established responsibilities while revising
the pinned integration contract and any behavior affected by the three
intervening releases.
This roadmap is based on PromptKit's pinned release guides for
[v0.6.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.6.0.md),
[v0.7.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.7.0.md),
and
[v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.8.0.md),
plus the public API and format documentation at the v0.8.0 tag.
## Target End State
- `go.mod` and `go.sum` pin PromptKit v0.8.0 without a local replacement or
vendored copy.
- Every maintained PromptKit prompt and profile prepares successfully under
v0.8.0's stricter validation and source-loading rules.
- Eligible Notarius structured completions use one PromptKit corrective call by
default after a structurally invalid response. Operators can explicitly set
a value from zero through three for a configured pipeline, with a more local
LLM-backed binding override where needed.
- PromptKit repair remains an inner operation within one Notarius stage
attempt. It never consumes or replenishes the binding's `retries` budget.
- A successful repaired result exposes cumulative usage and the actual repair
count to Notarius's application-owned response and debug models. A repaired
success is not itself a warning.
- Exhausted PromptKit validation remains an invalid structured-output result,
preserving the final candidate and diagnostics for debug and for any
applicable outer Notarius stage policy. Invalid structured output is never
accepted merely because the repair budget was exhausted.
- Profile inheritance, the built-in Rakestrawhome backend/profile, optional
credential behavior, and structured generation errors work through the
existing Notarius PromptKit boundary and are accurately documented.
- Provider-specific PromptKit types do not escape `internal/framework/llm`.
- Checkpoint identity, effective configuration, redacted summaries, and debug
provenance reflect every execution-affecting repair or profile change.
- Current documentation pins and describes v0.8.0; future Notarius semantic
validation retries remain roadmap behavior rather than being conflated with
this dependency upgrade.
## Release-by-Release Adoption
### PromptKit v0.6.0: Correctness, Safety, And Efficiency
PromptKit v0.6.0 adds no public declarations, but intentionally rejects several
formerly permissive or ambiguous inputs. The upgrade must audit Notarius's
embedded and operator-facing integration against these rules:
- YAML `id` and `version` metadata, rather than filenames, define prompt and
profile identity.
- Prompt `content_file` paths are exact, relative, contained paths; built-in
file artifacts must resolve to regular files.
- execution controls, output contracts, and repair budgets must be finite and
within their documented ranges;
- provider endpoints must be absolute HTTP or HTTPS URLs with a host and no
user information, query, or fragment;
- JSON documents and successful provider responses contain exactly one value;
- successful provider responses are bounded to 16 MiB; and
- JSON-compatible values are bounded for depth and expansion.
Notarius should rely on PromptKit for these rules rather than duplicate its
parsers or internal limits. Existing Notarius validation may retain a narrower
application rule where it has independent value, but overlapping validation
must agree with PromptKit and must not accept a value PromptKit will reject
later.
The upgrade automatically receives operation-local schema-plan reuse,
artifact-text memoization, improved cancellation checks, and transport error
identity preservation. Notarius should verify these changes through its real
adapter boundary and avoid adding a second cache or response-body layer that
would duplicate PromptKit's ownership.
### PromptKit v0.7.0: Profiles, Backend Access, And Generation Errors
#### Profile Inheritance
Operator profiles may use `base_profile` to alias or selectively refine a
built-in, fallback, or higher-precedence operator profile. Notarius must pass
profile sources through unchanged and let PromptKit own parent lookup, merge
rules, source precedence, cycle detection, and fully resolved prepared targets.
Preflight inspection must resolve inherited profiles through the same source
and backend composition used at execution. The selected leaf profile ID remains
the public profile identity, while effective backend, endpoint, model, and
reasoning provenance reflect the resolved chain. Notarius must not implement a
second inheritance parser.
The existing complete `dnd-extraction` fallback remains a standalone profile:
PromptKit v0.8.0 does not provide a built-in `openai/gpt-5.6-luna` profile that
would be an appropriate parent. Documentation should nevertheless explain how
operators can use inheritance for environment-specific workload profiles and
should link to PromptKit's pinned format contract rather than duplicate its
field-by-field merge algorithm.
Checkpoint safety must cover inherited behavior. Operator file/directory
digests already cover changes to definitions in those sources, fallback asset
digests cover application parents, and the PromptKit built-in catalog marker
must change from its v0.5.0 identity to v0.8.0 so a changed built-in parent
cannot reuse an incompatible checkpoint.
#### Rakestrawhome Backend And Profile
PromptKit's reserved `rakestrawhome` backend and
`rakestrawhome-gemma-4-31b` profile become available without Notarius-specific
registration. Notarius must not register or shadow the reserved backend ID.
Profile preflight, backend-capacity reporting, scheduling, generation, and
provenance should work for it through the same generic paths used by OpenRouter
and `local`.
The D&D default remains `dnd-extraction`; this upgrade does not silently move a
production workload to Rakestrawhome. Operator documentation should identify
the built-in profile as an available selection and link to PromptKit for its
endpoint, credential environment, model, and capacity defaults.
#### Optional Credentials
An absent or blank optional `APIKeyEnv` now causes PromptKit to omit the
`Authorization` header and send the request. Notarius must not restore the old
failure behavior by pre-reading provider credential environment variables or
by adding provider-specific authentication logic.
Profile inspection may report an explicit `APIKeyRequired` policy without
reading the credential, and execution remains the boundary at which that
requirement is enforced. For optional profiles, an authentication-requiring
provider may instead return a structured 401 or 403 generation failure. The
configuration and operations documentation must explain this distinction.
Notarius does not currently expose PromptKit's in-memory profile-registration
API to operators, and PromptKit's filesystem profile format does not expose
`APIKeyRequired`; therefore Notarius must not promise that an operator profile
can force local credential preflight. Operators should provision the named
environment variable, while Notarius should preserve the provider's structured
authentication failure when it is absent.
Notarius must continue to document mechanisms and environment-variable names,
never secret values.
#### Structured Generation Errors
The adapter should recognize `*promptkit.GenerationError` with `errors.As` and
translate useful information into an immutable, provider-neutral Notarius
error classification. At minimum, retain the HTTP status code so callers and
future retry policy can distinguish transport success with provider rejection
from other generation failures.
PromptKit's provider code, type, and message accessors are bounded but remain
untrusted and potentially sensitive. They must never appear automatically in
ordinary CLI output, warnings, manifests, checkpoint identity, or cache data.
If retained for an explicitly requested debug trace, they must pass through
Notarius's known-secret and bearer redaction and remain clearly identified as
untrusted provider diagnostics. Default error formatting should continue to
use a bounded, redacted application-owned message.
Capacity and cancellation retain their current more specific classifications
and precedence. This upgrade does not add automatic provider-error retry
classification; it only preserves safe structured data needed for diagnosis
and later policy.
### PromptKit v0.8.0: Bounded Structured-Output Repair
#### Default Policy
Every maintained production prompt whose output is consumed as structured data
should declare one repair attempt. All current production prompts use eligible
JSON Schema validation, so no current prompt needs a zero default merely
because of its output mode.
One repair means at most one corrective generation after the initial
candidate. PromptKit reconstructs the immutable original conversation and
appends only the latest invalid assistant candidate and latest deterministic
validation diagnostics. It preserves the selected target, direct session ID,
provider-native structured-output contract, and backend capacity policy. This
shape preserves the original cacheable prompt prefix and avoids accumulating
unbounded failed history.
The default is deliberately small. A single repair captures the common case in
which a capable model can correct malformed JSON or a schema violation after
receiving an exact diagnostic, while bounding the extra latency and cost of a
single structured completion.
#### Configuration Contract
The public configuration is an optional, presence-aware
`structured_output_repair_attempts` integer at pipeline scope and at each
LLM-backed module or validator binding. Its effective precedence is:
1. the binding value, when present;
2. the pipeline value, when present; and
3. the selected prompt's declared `repair_attempts` value.
The value must be from zero through three. Explicit zero disables PromptKit
repair at that scope. A deterministic binding must reject the field because it
cannot perform structured LLM repair. Validator bindings may use it only when
the selected validator is LLM-backed. Shorthand module bindings continue to
inherit the pipeline or prompt default.
The long, provider-neutral name is intentional: it distinguishes PromptKit's
inner structural repair from the existing binding `retries` field, which owns
complete stage attempts, without exposing a dependency name in generic
pipeline contracts.
The effective value must survive file parsing, cloning, redacted summaries,
pipeline resolution, and pipeline digest construction without pointer aliasing
or loss of presence. It must affect checkpoint identity because it can change
the selected result, latency, token usage, and provider cost.
#### Adapter Contract
The transport-neutral structured-completion request should carry an optional
application-owned structural-repair budget. No `promptkit.OutputContract` or
other PromptKit type may cross the adapter boundary.
PromptKit v0.8.0 request validation replaces the complete prompt output
contract rather than merging one field. When Notarius has a configured
override, the adapter must therefore inspect the selected prompt, copy its
normalized declared format, validation mode, and schema path, change only the
repair count, and supply that complete contract on the prepared request. A nil
override continues to use the prompt declaration directly. Inspection and
preparation must use the same immutable engine sources; a small adapter-local
cache keyed by normalized prompt ID and version is acceptable but not required
without measured need.
This approach prevents configuration from accidentally dropping JSON Schema
validation, avoids duplicating schema paths in pipeline YAML, and keeps prompt
assets authoritative for every output-contract field other than the explicit
operator override.
The transport-neutral structured-completion response should report the actual
number of PromptKit repair calls. PromptKit's returned token usage is already
cumulative and must be passed through without re-summing it. Debug records
should distinguish the configured budget from the actual count. Ordinary run
manifests need not gain raw prompt or response data merely to report repairs;
any durable aggregate should be added only if it has a clear consumer contract.
#### Result And Failure Semantics
- A valid initial candidate returns normally with zero actual repairs.
- A valid corrected candidate returns normally with cumulative usage and its
positive actual repair count. It does not emit a warning solely because a
repair occurred.
- Exhausting the repair budget returns PromptKit's final candidate and failed
validation result. The adapter maps this to
`ErrInvalidStructuredOutput`, preserves the response and debug material, and
does not decode or accept the candidate.
- An explicitly empty or whitespace-only candidate participates in the
declared structural validation and repair flow. Missing, `null`, or
non-string provider content remains a malformed provider response.
- A generation failure during a corrective call is an operational generation
failure and uses the same safe structured-error adaptation as an initial
generation failure.
- Context cancellation remains authoritative throughout the initial and
corrective calls.
PromptKit repair happens inside one scheduled `CompleteStructured` operation.
The Notarius scheduler holds one permit for that logical operation while
PromptKit performs its initial and serial corrective calls; PromptKit
reacquires its own selected-backend capacity for each corrective generation.
Because corrective calls are serial, this cannot expand actual concurrent
provider work beyond the number of admitted Notarius operations, but
documentation must stop describing the Notarius permit as a separate admission
event for every internal repair call.
One `CompleteStructured` invocation with effective PromptKit repair budget `R`
may make at most `R + 1` provider calls. If one stage attempt makes `C`
structured-completion invocations, a binding with `retries: N` has an upper
bound of `(N + 1) * C * (R + 1)` provider calls; `C` may itself be a bounded,
data-dependent module property, as it is for batched semantic reconciliation.
LLM-backed validators have their own corresponding invocation counts, budgets,
and costs. These formulas are upper bounds, not promises that every failure is
retryable or that every attempt reaches the provider.
## Profile And Prompt Source Compatibility
The upgrade must preserve Notarius's source precedence: an operator source,
then registered application fallback profiles, then PromptKit built-ins. A
selected malformed definition remains authoritative and fails rather than
falling through. Parent resolution introduced by profile inheritance observes
that same precedence.
All embedded prompt manifests, shared content fragments, response schemas, and
fallback profiles must be prepared or inspected offline under v0.8.0. The
review should specifically catch:
- IDs inferred accidentally from filenames;
- stale or escaping `content_file` paths;
- missing or non-regular embedded artifacts;
- repair values outside zero through three or paired with ineligible
validation;
- schemas or examples that are not exact single JSON documents;
- unsupported endpoint forms; and
- JSON-compatible variables or profile extras that exceed upstream bounds.
No prompt prose, schema shape, durable D&D artifact contract, or default D&D
model should change merely to exercise the dependency. Prompt manifests should
change only as needed to enable the adopted repair default and satisfy v0.8.0
contracts.
## Provenance, Debugging, And Security
- Update the opaque PromptKit built-in profile-catalog identity from v0.5.0 to
v0.8.0. Do not hash or publish PromptKit's internal catalog bytes.
- Ensure a prompt's repair default remains covered by its existing prompt asset
fingerprint and a configured effective override remains covered by the
resolved pipeline digest.
- Preserve selected leaf profile identity while recording the inherited
effective target already exposed by PromptKit inspection and prepared
details.
- Add actual structural-repair count and, when useful, the configured budget to
application-owned debug material. Token totals remain PromptKit's cumulative
values.
- Do not generate a warning for a successful repair. Repair exhaustion is an
invalid-output failure, while provider rejection is a generation failure.
- Never expose raw provider diagnostic fields without explicit debug capture
and application redaction. Do not place them in normal errors or durable
summaries.
- Preserve context and transport error identity sufficiently for
`errors.Is`-based cancellation and deadline handling after adapting the
external error.
## Documentation And Examples
Implementation should update current-state documentation only when the new
behavior lands:
- `docs/integrations/pkg-promptkit.md` must pin v0.8.0 and define the revised
prepared-execution, repair, profile-inheritance, backend, credential, and
error-adaptation boundary.
- `docs/config.md` must own the repair configuration fields, precedence,
allowed range, explicit-zero behavior, profile inheritance availability, and
optional credential semantics.
- `docs/operations.md` must explain structural repair cost, timeout and
concurrency effects, credential failures, and its distinction from stage
retries.
- `docs/internal/llm.md` must describe adapter contract replacement, actual
repair metadata, error adaptation, source compatibility, and scheduling.
- `docs/internal/pipeline.md` must describe how effective repair configuration
is resolved and how inner repair differs from outer stage attempts.
- `docs/policy/architecture.md` should receive only the durable ownership rule:
PromptKit owns bounded deterministic structural repair within one completion,
while Notarius owns stage attempts and semantic validation policy. Detailed
fields and retry formulas belong in their canonical configuration and
operations documents.
Update maintained configuration examples only if the public Notarius
configuration contract changes. A short inheritance illustration may remain in
the configuration reference; do not create a complete example solely to copy
PromptKit's upstream profile catalog. All upstream links must point to the
v0.8.0 tag. Historical release or archived roadmap references should remain
historical.
No ADR is required solely to pin a newer dependency. The durable separation
between PromptKit structural repair and Notarius semantic stage retries should
be stated in architecture documentation now; the more extensive future
validation state machine still warrants the separate ADR already identified in
`future.md` when that work is promoted.
## Validation And Acceptance Criteria
The implementation is complete when:
- the repository builds and tests against PromptKit v0.8.0 with no replacement
directive, workspace dependency, or vendored source;
- every maintained prompt and profile prepares or inspects successfully under
the v0.8.0 source, path, endpoint, output-contract, and JSON-value rules;
- an invalid first JSON Schema candidate followed by a valid correction returns
the valid raw output, cumulative usage, and actual repair count through the
Notarius adapter;
- repair exhaustion returns the final raw candidate and debug material with an
error matching `ErrInvalidStructuredOutput`;
- a corrective generation failure retains safe generation classification and
provider status without leaking untrusted provider detail;
- explicit empty content follows structural validation rather than being
misclassified by Notarius;
- repair configuration is presence-aware, range checked, rejected on
deterministic bindings, resolved with documented precedence, and included in
effective pipeline identity;
- inherited profiles resolve consistently during preflight and execution, and
changes to any relevant operator, fallback, or built-in parent invalidate
checkpoint reuse;
- the Rakestrawhome built-in profile reaches generic preflight, scheduling, and
provenance paths without application-specific registration;
- optional missing credentials and explicitly required credentials behave as
documented without contacting real providers in tests;
- cancellation, timeout, backend capacity, prepared-execution snapshot,
session ID, raw-output, debug-redaction, and existing profile provenance
behavior remain intact;
- maintained examples validate successfully; and
- canonical documentation contains no active v0.5.0 pin or claim that PromptKit
is always single-pass.
Tests should follow `docs/policy/testing.md`: exercise observable Notarius
contracts with offline fake clients or `httptest` boundaries, and do not copy
PromptKit's entire internal repair test suite or assert its exact correction
message prose. The dependency's internal wording is not a Notarius contract.
## Non-Goals
- Implementing Notarius's future feedback-aware semantic stage-retry loop.
- Adding the D&D combat-scene semantic validator.
- Redesigning warning policy or treating successful structural repair as a
warning.
- Adding provider transport retries or deciding which HTTP statuses should
consume a stage retry.
- Exposing PromptKit request, response, profile, validation, capacity, or error
types outside the LLM adapter.
- Changing durable artifact schemas, D&D prompt semantics, the D&D default
model, or the fixed pipeline shape.
- Reimplementing PromptKit profile inheritance, schema validation, response
bounds, repair conversations, backend admission, or provider parsing inside
Notarius.
## Decisions
### 1. Default Structured-Output Repair Budget
**Decision: default to one repair attempt.** Set every maintained
eligible production prompt to `repair_attempts: 1`. One corrective call is a
strong fit for Notarius because every current production LLM response has a
strict JSON Schema contract, smaller cost-effective models are a deliberate
deployment target, and a precise structural diagnostic often makes one retry
materially more successful. The budget is paid only after a structurally
invalid candidate and remains tightly bounded.
**Alternative considered: retain zero by default.** This preserves single-pass
cost and latency and requires operators to opt in. It is preferable for an
environment where every additional request is expensive or where upstream
provider-native schema enforcement already produces negligible invalid output.
It is less suitable as the Notarius default because one malformed response can
otherwise discard substantial completed pipeline work.
**Alternative considered: default to two.** This may improve recovery for
weak models, but it doubles the worst-case corrective cost relative to the
selected default and compounds with outer stage retries. It should be an
operator choice supported by configuration, not the initial default, unless
observational evidence shows that the second correction has a worthwhile
marginal success rate.
### 2. Repair Override Scope
**Decision: support both pipeline and LLM-backed binding overrides.** Use
the presence-aware `structured_output_repair_attempts` field and precedence
defined above. A pipeline value provides the convenient one-line control the
operator requested, while a binding value permits an expensive normalizer or
future LLM-backed validator to use a deliberately different budget. This
mirrors Notarius's established pipeline/binding profile inheritance and scales
without editing embedded prompts.
**Alternative considered: support only a pipeline override.** This is smaller to
implement and document and still permits global enablement or disablement for
one pipeline. Its drawback is that one exceptional prompt cannot opt out or
request a larger budget without changing an embedded asset for every pipeline.
**Alternative considered: expose one global value under the top-level
`promptkit` configuration.** This makes client construction simple, but applies
the same budget to unrelated pipelines and leaks an execution policy into the
dependency configuration block. It is less compositional than pipeline-owned
policy and therefore not recommended.
### 3. Retention Of Provider-Supplied Generation Details
**Decision: retain status in the application-owned error contract and
retain redacted provider code, type, and message only in explicitly requested
debug traces.** Status is useful for diagnosis and future retry policy without
usually containing sensitive data. The other fields can materially explain a
400 response but may echo request or schema content, so they belong only in the
already-sensitive debug surface after Notarius redaction.
**Alternative considered: retain only HTTP status and discard all provider fields.**
This is the safest and smallest policy and still improves typed failure
handling. It sacrifices potentially decisive provider diagnostics, leaving an
operator with less information when a provider returns a terse status and the
problem cannot be reproduced easily.
**Alternative considered: include bounded provider code and type in normal
errors while keeping message debug-only.** Codes and types are often stable and
less sensitive than messages, but PromptKit explicitly classifies every
provider field as untrusted. Promoting them to ordinary output creates a
disclosure and compatibility burden that is not currently justified.

View File

@@ -1,282 +0,0 @@
# Source-Only Releases
## Status
Implemented. Creating the first release under this procedure remains a
separate maintainer operation.
## Purpose
Define a repeatable, guarded release process for Notarius without taking on a
binary-distribution system that its current operator audience does not need.
The process should make an exact source revision, its compatibility impact,
and its validation status easy to identify while keeping installation in the
hands of technically capable operators and deployment automation.
The model is adapted from Weatherreporter's release procedure, but its target
is deliberately narrower: an immutable source tag and checked-in release note
are the release. Notarius does not publish executable archives or support
Windows as part of this work.
## Release Model
Notarius releases come from commits on `main` and use stable semantic-version
tags in the form `vMAJOR.MINOR.PATCH`. Prerelease tags are not part of the
initial process.
Every release has one nonempty, version-matched note at
`docs/releases/<tag>.md`. The note and every affected current-state document
must be present in the tagged commit. The Git tag and checked-in note together
are the durable release record; no separately editable release page is
required.
Published tags are immutable. A maintainer must never move, reuse, or delete a
published tag. If a published candidate is defective, the correction is made
on `main` and released under a new patch version. An unpublished local tag may
be deleted when candidate inspection finds a problem before any remote push.
Before `v1.0.0`, a minor release may intentionally change a documented CLI,
configuration, durable artifact, integration, or operating contract when its
release note explains the impact and required operator action. A patch release
must not intentionally break those documented contracts within its minor
line.
The existing `v0.1.0`, `v0.2.0`, and `v0.3.0` tags remain unchanged. They
predate this procedure and do not need retrospective release notes. The first
release made under this process establishes the release-note series.
## Source-Only Distribution
Notarius does not publish release binaries, archives, installers, container
images, package-manager entries, checksum files, or signatures. A release tag
is suitable for Go-native installation and for an operator-controlled build
from an exact checkout.
The primary installation form is:
```sh
GOWORK=off go install \
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
```
Operator documentation should also describe cloning the repository, checking
out the tag in detached-head state, and building `./cmd/notarius` with the Go
version declared by `go.mod`. Private-module authentication and `GOPRIVATE`
configuration belong to the operator environment and must be documented by
mechanism rather than with real credentials.
Consumers such as Narratio should pin the desired Notarius tag in provisioning
or deployment configuration. They must continue to decide runtime
compatibility from Notarius's published receipt and artifact schema contracts,
not merely from the executable's product version.
Packaged binaries may be reconsidered if distribution demand, installation
friction, or a broader user audience justifies their build, signing, retention,
and platform-support costs. They are not a prerequisite for a disciplined
release process.
## Platform Policy
Linux is the supported deployment platform. Release validation must run the
test suite and the release build on Linux and must confirm that the command
builds with `CGO_ENABLED=0` for Linux `amd64` and `arm64`.
macOS is a best-effort development and testing platform. Release validation
should confirm that the command cross-compiles with `CGO_ENABLED=0` for Darwin
`amd64` and `arm64`, but the project does not promise packaged artifacts or a
separate runtime test environment for those targets.
Windows is unsupported. The release process must not require Windows builds,
Windows-specific compatibility work, or Windows documentation. Platform-
specific implementation may intentionally use Unix facilities when they are
important to Notarius's filesystem safety and operational model. Any later
decision to support Windows requires its own feature scope and validation
policy.
## Version Reporting
Add a root `notarius --version` interface for deployment diagnostics. It
prints exactly one line:
```text
notarius vMAJOR.MINOR.PATCH
```
when the build has a valid release version, and:
```text
notarius development
```
when no release version is available.
The implementation must obtain the main-module version from Go build
information so `go install ...@vMAJOR.MINOR.PATCH` reports the selected tag. It
must also accept an optional link-time version override so controlled builds
and release CI can identify an exact tag from a checkout. The override must be
validated and must not silently turn arbitrary text into a release version.
Ordinary unversioned checkout builds remain `development`; the release process
must not modify a tracked source constant for each release.
Version reporting is an informational product interface. It does not replace
receipt, configuration, prompt, or artifact schema versioning, and it must not
be used as the sole downstream compatibility check.
## Release Notes
Each new `docs/releases/<tag>.md` document has this minimum structure:
```markdown
# Notarius vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
The note should concisely explain the release's purpose, compatibility with the
preceding release, operator actions, and material user-visible, operational,
integration, and maintainer-visible changes. It should link to canonical
current-state documentation for exact contracts rather than duplicating those
contracts.
Release notes are durable historical summaries. They must not contain
credentials, private infrastructure detail, sensitive campaign material, or
claims that are not true of the tagged candidate. A release note does not
excuse stale current-state documentation; affected canonical documents are
updated in the same candidate.
## Candidate Validation
The release procedure must provide copyable POSIX-shell guards that validate
the release version, release-note filename and heading, required note sections,
repository state, and module hygiene. Validation must be run from the Notarius
repository root with Go workspace behavior disabled.
At minimum, a candidate must pass:
- no tracked `go.work` or `go.work.sum`, no vendored tree, and no `replace`
directive in `go.mod`;
- `GOWORK=off go test -count=1 ./...`;
- `GOWORK=off go test -race -count=1 ./...`;
- `GOWORK=off go vet ./...`;
- `GOWORK=off go build ./...`;
- `GOWORK=off go mod tidy -diff`;
- `gofmt` verification for every tracked Go file;
- `git diff --check` and `git diff --cached --check`;
- validation of both maintained D&D configuration examples with their selected
pipeline;
- Linux `amd64` and `arm64` static command builds;
- best-effort Darwin `amd64` and `arm64` static command builds; and
- a focused manual or automated check that every added or changed local
Markdown link resolves.
The candidate review also checks for generated binaries, test output,
credentials, temporary files, module replacements, vendored dependencies, and
other unintended source-control content. Tests remain offline and do not call
an LLM provider or require live credentials.
## Candidate Publication
The release procedure must guard the exact commit immediately before tagging.
It requires:
- the current branch is `main`;
- the worktree and index are clean;
- the candidate commit has been pushed and exactly matches `origin/main`;
- the matching release note exists in that commit;
- no local or remote tag already uses the selected version; and
- the substantive release checks have passed for that exact candidate.
The maintainer records the exact candidate commit, creates a lightweight tag
bound explicitly to that commit, verifies the local tag target, and pushes only
that tag ref. The procedure must not recommend `git push --tags`.
After publication, the maintainer verifies that the remote tag resolves to the
guarded commit and that the release note can be read from the tagged tree. A
fresh temporary checkout or `go install ...@<tag>` must build successfully, and
the resulting command must report the expected version through `--version`.
## Validation-Only Release Automation
Add a tag-triggered Woodpecker pipeline that validates source releases without
publishing artifacts. It should:
- accept only stable semantic-version tags;
- require the version-matched release note;
- run the same substantive module, test, race, vet, build, formatting, and
whitespace checks as the documented local procedure;
- validate the maintained configuration examples;
- perform the supported and best-effort cross-build checks; and
- verify a release-version build's `notarius --version` output on the CI host.
The pipeline must not upload binaries, create archives or checksums, create or
edit a Gitea release object, or require a release API token. Local guards remain
authoritative before tag publication because CI begins only after the tag is
already remote.
If tag validation fails, preserve the published tag, fix the cause on `main`,
select a new patch version, and repeat the full process. Do not weaken tag
immutability merely because the release contains source rather than binaries.
## Documentation Ownership
In the target state:
- `docs/release.md` owns the maintainer release procedure, commands, ordering,
publication checks, and failure recovery;
- `docs/releases/` owns one historical summary per release made under the new
process;
- `docs/cli.md` owns the `--version` contract;
- `README.md` owns the shortest source-installation example and links to the
release procedure where useful;
- `docs/development.md` routes release preparation, tagging, and verification
work to `docs/release.md`;
- `docs/policy/documentation.md` assigns canonical ownership to the release
procedure and release notes;
- `docs/policy/architecture.md` records Linux support, best-effort macOS
development, unsupported Windows, and source-only distribution only if those
are judged durable development invariants rather than release mechanics; and
- `docs/operations.md` describes only installation or deployment consequences
relevant to operators and links to canonical CLI and release contracts.
Current-state documentation must not describe the new release process,
`--version`, or automated validation until the corresponding behavior exists.
## Acceptance Criteria
- A maintainer can prepare, validate, tag, publish, and verify a source release
by following `docs/release.md` without relying on undocumented knowledge.
- Every new release has an immutable semantic-version tag and matching
checked-in release note in the tagged commit.
- The guarded candidate is clean, synchronized with `origin/main`, and passes
the documented substantive checks before tagging.
- Tag-triggered CI independently validates the published source and never
publishes binary artifacts.
- `go install` of a tagged version succeeds and `notarius --version` reports
that version; ordinary unversioned builds report `development`.
- Linux is the documented supported deployment platform, macOS has a
best-effort development build check, and Windows is explicitly unsupported.
- Downstream compatibility remains based on durable Notarius contracts rather
than the product version alone.
- Existing pre-procedure tags remain untouched and require no invented release
history.
## Non-Goals
- Publishing executable archives, installers, container images, checksums,
signatures, or package-manager entries.
- Supporting or cross-compiling for Windows.
- Creating or maintaining a mutable Gitea release page.
- Supporting prerelease tag syntax in the initial procedure.
- Automating version selection, release-note authorship, commits, or tag
creation.
- Retrospectively creating release notes for `v0.1.0` through `v0.3.0`.
- Treating a product version as a substitute for receipt, configuration,
prompt, or artifact schema compatibility.

View File

@@ -1,6 +1,9 @@
version: 4 version: 5
configkit:
imports:
- path: pipelines/dnd-session.yml
promptkit: promptkit:
profile_file: ./examples/profiles/dnd-extraction.yml profile_file: ./llm-profiles/dnd-extraction.yml
concurrency: concurrency:
total_llm: 2 total_llm: 2
stage_workers: stage_workers:
@@ -16,145 +19,3 @@ cache:
directory: ./notarius-cache/checkpoints directory: ./notarius-cache/checkpoints
debug: debug:
directory: ./notarius-debug directory: ./notarius-debug
pipelines:
dnd-session:
llm_profile: dnd-extraction
input: seriatim
# Stable campaign context is shared by every module that accepts these slots.
references:
party: ./dnd-party.txt
glossary: ./dnd-glossary.txt
chunk:
module: dnd/scenes
retries: 2
output:
module: json
options:
include_chunk_map: true
evidence_context:
enabled: true
window_units: 3
lanes:
- item-occurrences
- item-registry
- location-registry
- location-occurrences
- npc-registry
- spells
- combat-turns
- npc-occurrences
- enemy-events
steps:
# Establish session-wide reference artifacts before their consumers.
- id: describe-session
artifacts:
item-registry:
extract:
module: dnd/item-registry
retries: 2
merge: appendorder
normalize: dnd/item-registry
npc-registry:
extract:
module: dnd/npc-registry
retries: 2
merge: appendorder
normalize:
module: dnd/npc-registry
retries: 2
location-registry:
extract:
module: dnd/location-registry
retries: 2
merge: appendorder
normalize:
module: dnd/location-registry
retries: 2
scene-descriptions:
extract:
module: dnd/scene-descriptions
retries: 2
merge: appendorder
normalize: dnd/scene-descriptions
- id: extract-events
# Accepted registry artifacts and scene-description eligibility artifacts
# are supplied in memory to their compatible consumers in this step.
references:
location_registry:
artifact:
step: describe-session
lane: location-registry
npc_registry:
artifact:
step: describe-session
lane: npc-registry
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
item_registry:
artifact:
step: describe-session
lane: item-registry
artifacts:
item-occurrences:
extract:
module: dnd/item-occurrences
retries: 2
merge: appendorder
normalize: dnd/item-occurrences
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog: ./dnd-spell-catalog.json
merge: appendorder
# Stage-local file references are intentionally bound at each stage.
normalize:
module: dnd/spells
references:
spell_catalog: ./dnd-spell-catalog.json
combat-turns:
extract:
module: dnd/combat-turns
retries: 2
merge: appendorder
normalize: dnd/combat-turns
npc-occurrences:
extract:
module: dnd/npc-occurrences
retries: 2
merge: appendorder
normalize: dnd/npc-occurrences
location-occurrences:
extract:
module: dnd/location-occurrences
retries: 2
merge: appendorder
normalize: dnd/location-occurrences
- id: track-enemies
references:
npc_registry:
artifact:
step: describe-session
lane: npc-registry
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
combat_turns:
artifact:
step: extract-events
lane: combat-turns
npc_occurrences:
artifact:
step: extract-events
lane: npc-occurrences
artifacts:
enemy-events:
extract:
module: dnd/enemy-events
retries: 2
merge: appendorder
normalize: dnd/enemy-events

View File

@@ -1,9 +1,12 @@
version: 4 version: 5
pipelines: pipelines:
dnd-session: dnd-session:
llm_profile: dnd-extraction llm_profile: dnd-extraction
input: seriatim input:
module: seriatim
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
normalize: dnd/spells module: dnd/spells
normalize:
module: dnd/spells

View File

@@ -0,0 +1,164 @@
pipelines:
dnd-session:
llm_profile: dnd-extraction
input:
module: seriatim
# Stable campaign context is shared by every module that accepts these slots.
references:
party:
path: ../dnd-party.txt
glossary:
path: ../dnd-glossary.txt
chunk:
module: dnd/scenes
retries: 2
output:
module: json
options:
include_chunk_map: true
evidence_context:
enabled: true
window_units: 3
lanes:
- item-occurrences
- item-registry
- location-registry
- location-occurrences
- npc-registry
- spells
- combat-turns
- npc-occurrences
- enemy-events
steps:
# Establish session-wide reference artifacts before their consumers.
- id: describe-session
artifacts:
item-registry:
extract:
module: dnd/item-registry
retries: 2
merge:
module: appendorder
normalize:
module: dnd/item-registry
npc-registry:
extract:
module: dnd/npc-registry
retries: 2
merge:
module: appendorder
normalize:
module: dnd/npc-registry
retries: 2
location-registry:
extract:
module: dnd/location-registry
retries: 2
merge:
module: appendorder
normalize:
module: dnd/location-registry
retries: 2
scene-descriptions:
extract:
module: dnd/scene-descriptions
retries: 2
merge:
module: appendorder
normalize:
module: dnd/scene-descriptions
- id: extract-events
# Accepted registry artifacts and scene-description eligibility artifacts
# are supplied in memory to their compatible consumers in this step.
references:
location_registry:
artifact:
step: describe-session
lane: location-registry
npc_registry:
artifact:
step: describe-session
lane: npc-registry
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
item_registry:
artifact:
step: describe-session
lane: item-registry
artifacts:
item-occurrences:
extract:
module: dnd/item-occurrences
retries: 2
merge:
module: appendorder
normalize:
module: dnd/item-occurrences
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog:
path: ../dnd-spell-catalog.json
merge:
module: appendorder
# Stage-local file references are intentionally bound at each stage.
normalize:
module: dnd/spells
references:
spell_catalog:
path: ../dnd-spell-catalog.json
combat-turns:
extract:
module: dnd/combat-turns
retries: 2
merge:
module: appendorder
normalize:
module: dnd/combat-turns
npc-occurrences:
extract:
module: dnd/npc-occurrences
retries: 2
merge:
module: appendorder
normalize:
module: dnd/npc-occurrences
location-occurrences:
extract:
module: dnd/location-occurrences
retries: 2
merge:
module: appendorder
normalize:
module: dnd/location-occurrences
- id: track-enemies
references:
npc_registry:
artifact:
step: describe-session
lane: npc-registry
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
combat_turns:
artifact:
step: extract-events
lane: combat-turns
npc_occurrences:
artifact:
step: extract-events
lane: npc-occurrences
artifacts:
enemy-events:
extract:
module: dnd/enemy-events
retries: 2
merge:
module: appendorder
normalize:
module: dnd/enemy-events

View File

@@ -0,0 +1,8 @@
# This selects operational settings only; choose a different LLM profile with
# the pipeline's llm_profile field or the run command's --llm-profile flag.
concurrency:
total_llm: 1
stage_workers:
extract: 1
output:
directory: ./notarius-output-development

11
go.mod
View File

@@ -3,9 +3,16 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5 go 1.25.5
require ( require (
gitea.maximumdirect.net/eric/promptkit v0.8.0 gitea.maximumdirect.net/eric/ConfigKit v0.3.0
gitea.maximumdirect.net/eric/promptkit v0.10.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1 // indirect
) )
require golang.org/x/text v0.40.0 require golang.org/x/text v0.40.0
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
)

12
go.sum
View File

@@ -1,9 +1,17 @@
gitea.maximumdirect.net/eric/promptkit v0.8.0 h1:NGd9hDLu0UMxKbvittMrqM5Ua94eFb+kOE7UIir8l08= gitea.maximumdirect.net/eric/ConfigKit v0.3.0 h1:LgXEUjR0KsgwGBznOWa8FJTQ+iElctHtI1via5vKdu4=
gitea.maximumdirect.net/eric/promptkit v0.8.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ= gitea.maximumdirect.net/eric/ConfigKit v0.3.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/promptkit v0.10.0 h1:Nyi+YbqvO7tkRjen+Opc79wuzuKIIwcSfkriixPPwrE=
gitea.maximumdirect.net/eric/promptkit v0.10.0/go.mod h1:0zEBVlJJC+ldDdXazAgybYBXNSe74vwFx5dyOuSos/w=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1 h1:S2/Q/JDE+m8Ca3pDhAjVbfYrdlEhs9blj0STgNopHo0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0 h1:Y0EXc3eh12eKW2NrOviVqdk3oBo6UUjopt1YAa1B3Oc=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

View File

@@ -21,6 +21,10 @@ import (
const assembledSpellExtractorKey = "test/dnd/spell-casts" const assembledSpellExtractorKey = "test/dnd/spell-casts"
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{}) registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
@@ -67,28 +71,39 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct) t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
} }
wantWarningReasons := []string{ wantDiagnosticReasons := []string{
spellnormalize.ReasonCodeSpellNameCanonicalized, spellnormalize.ReasonCodeSpellNameCanonicalized,
spellnormalize.ReasonCodeSourceReferencesNormalized, spellnormalize.ReasonCodeSourceReferencesNormalized,
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed, spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
"spell_not_near_source", "spell_not_near_source",
} }
gotWarningReasons := make([]string, len(output.Warnings)) gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
for index, warning := range output.Warnings { for index, group := range output.Diagnostics.Groups {
gotWarningReasons[index] = warning.ReasonCode gotDiagnosticReasons[index] = group.ReasonCode
} }
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) { if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings) t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
} }
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") { if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2]) t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
} }
warningsFile := decodeAssembledOutput[struct { warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"` Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json") }](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) { if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings) t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Truncated bool `json:"truncated"`
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
} }
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json") manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key { if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
@@ -101,6 +116,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
} }
} }
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
}
correction := extractor.correctionSnapshot()
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
}
}
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) { func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true}) registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
var normalizeChain *pipeline.ResolvedValidatorChain var normalizeChain *pipeline.ResolvedValidatorChain
@@ -127,15 +166,16 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output) t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
} }
for _, warning := range output.Warnings { for _, group := range output.Diagnostics.Groups {
if warning.ReasonCode == "spell_not_near_source" { if group.ReasonCode == "spell_not_near_source" {
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings) t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
} }
} }
} }
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) { func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true}) registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil { if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err) t.Fatalf("Prepare() error = %v, want nil", err)
@@ -161,12 +201,12 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) { if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected) t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
} }
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" { if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want terminal normalize catalog warning", output.Warnings) t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
} }
} }
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) { func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true}) registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil { if err != nil {
@@ -190,14 +230,20 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" { if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts) t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
} }
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" { if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings) t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
} }
warningsFile := decodeAssembledOutput[struct { warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"` Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json") }](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) { if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings) t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
} }
} }
@@ -206,6 +252,47 @@ type assembledSpellPipelineOptions struct {
unknownSpell bool unknownSpell bool
} }
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
extractor := &assembledCorrectingSpellExtractor{}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledCorrectingSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register correcting extractor: %v", err)
}
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
return assembledDirectSpellValidator{}, nil
}); err != nil {
t.Fatalf("register direct spell validator: %v", err)
}
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
extract.Retries = 1
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
},
Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return components.registries, resolved, extractor
}
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) { func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
t.Helper() t.Helper()
components := productionTestComponents(t) components := productionTestComponents(t)
@@ -251,6 +338,69 @@ type assembledSpellExtractor struct {
unknownSpell bool unknownSpell bool
} }
type assembledCorrectingSpellExtractor struct {
mu sync.Mutex
correction *contracts.SemanticCorrection
}
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if req.Source == nil || req.Chunk == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
}
response := `{"spell":"accepted"}`
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
if req.Chunk.Index == 0 && req.Correction == nil {
response = `{"spell":"Mysterious Burst"}`
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
if req.Chunk.Index == 0 && req.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
e.mu.Lock()
e.correction = correction
e.mu.Unlock()
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
}
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
e.mu.Lock()
defer e.mu.Unlock()
correction, err := contracts.CloneSemanticCorrection(e.correction)
if err != nil {
return nil
}
return correction
}
type assembledDirectSpellValidator struct{}
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
for _, cast := range req.Value.SpellCasts {
if cast.Spell == "Mysterious Burst" {
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
}
}
return contracts.ValidationResult{Approved: true}, nil
}
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey } func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
@@ -296,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...) return append([]int(nil), e.chunkIndexes...)
} }
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount
}
return count
}
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T { func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
t.Helper() t.Helper()
for _, file := range files { for _, file := range files {

View File

@@ -23,7 +23,7 @@ func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
}{ }{
{name: "default", wantStores: 1}, {name: "default", wantStores: 1},
{name: "file", fileMode: "bypass"}, {name: "file", fileMode: "bypass"},
{name: "environment", envMode: "bypass"}, {name: "retired environment has no effect", envMode: "bypass", wantStores: 1},
{name: "cli", envMode: "refresh", cliMode: "bypass"}, {name: "cli", envMode: "refresh", cliMode: "bypass"},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -80,25 +80,14 @@ func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
}) })
for _, tt := range []struct { for _, tt := range []struct {
name string name string
fileConfig bool
}{ }{
{name: "invalid environment mode"}, {name: "invalid file mode"},
{name: "invalid file mode", fileConfig: true},
} { } {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
opts := newStateTestHarness().options() opts := newStateTestHarness().options()
if tt.fileConfig { replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
} else {
opts.LookupEnv = func(name string) (string, bool) {
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" {
return "invalid", true
}
return "", false
}
}
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts) code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
@@ -207,16 +196,20 @@ func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
configText := replaceRequiredOnce(t, string(data), " chunk: test/chunk\n", ` chunk: configText := replaceRequiredOnce(t, string(data), " chunk:\n module: test/chunk\n", ` chunk:
module: test/chunk module: test/chunk
options: options:
strategy: first strategy: first
`) `)
configText = replaceRequiredOnce(t, configText, " output: test/output\n", ` other: configText = replaceRequiredOnce(t, configText, " output:\n module: test/output\n", ` other:
extract: test/extract extract:
merge: test/merge module: test/extract
normalize: test/normalize merge:
output: test/output module: test/merge
normalize:
module: test/normalize
output:
module: test/output
`) `)
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -140,7 +140,7 @@ func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
stdout.Reset() stdout.Reset()
stderr.Reset() stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t)) code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t))
if code != 1 || !strings.Contains(stderr.String(), malformed) || !strings.Contains(stderr.String(), "parse config file") || stdout.Len() != 0 { if code != 1 || !strings.Contains(stderr.String(), "configuration contains invalid YAML") || stdout.Len() != 0 {
t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
} }
} }
@@ -185,14 +185,16 @@ func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) {
} }
func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) { func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) {
configPath := writeCommandConfigContent(t, `version: 4 configPath := writeCommandConfigContent(t, `version: 5
pipelines: pipelines:
demo: demo:
llm_profile: dnd-extraction llm_profile: dnd-extraction
input: seriatim input:
module: seriatim
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
module: dnd/spells
`) `)
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{}) code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{})
@@ -261,19 +263,21 @@ func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string,
func writeCommandConfig(t *testing.T, firstID, secondID string) string { func writeCommandConfig(t *testing.T, firstID, secondID string) string {
t.Helper() t.Helper()
content := fmt.Sprintf("version: 4\npipelines:\n %q:\n input: seriatim\n %q:\n input: seriatim\n", firstID, secondID) content := fmt.Sprintf("version: 5\npipelines:\n %q:\n input:\n module: seriatim\n %q:\n input:\n module: seriatim\n", firstID, secondID)
return writeCommandConfigContent(t, content) return writeCommandConfigContent(t, content)
} }
func writeResolvableCommandConfig(t *testing.T) string { func writeResolvableCommandConfig(t *testing.T) string {
t.Helper() t.Helper()
return writeCommandConfigContent(t, `version: 4 return writeCommandConfigContent(t, `version: 5
pipelines: pipelines:
demo: demo:
input: seriatim input:
module: seriatim
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
module: dnd/spells
`) `)
} }

View File

@@ -0,0 +1,157 @@
package cli
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config effective", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
if err := loaded.WriteEffective(stdout); err != nil {
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
return 1
}
return 0
}
func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config explain", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 1 {
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
return 2
}
pointer := fs.Arg(0)
parsed, err := configkit.ParsePath(pointer)
if err != nil || parsed.String() != pointer {
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
if err := loaded.WriteExplanation(stdout, pointer); err != nil {
fmt.Fprintf(stderr, "notarius: write configuration explanation: %v\n", err)
return 1
}
return 0
}
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
pipelineID := fs.String("pipeline", "", "pipeline ID")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
if strings.TrimSpace(*onlyRaw) != "" && strings.TrimSpace(*pipelineID) == "" {
fmt.Fprintln(stderr, "notarius: --only requires --pipeline")
return 2
}
only, err := parseOnly(*onlyRaw)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
loaded, path, err := loadConfigKit(*configPath, *profile, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
if strings.TrimSpace(*pipelineID) == "" {
fmt.Fprintf(stdout, "config %q is valid\n", path)
return 0
}
catalog, err := effectiveCatalog(opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
effective, err := loaded.Config.Resolve(config.ResolveInput{PipelineID: *pipelineID, Only: only, Catalog: catalog})
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "config %q is valid for pipeline %q\n", path, strings.TrimSpace(*pipelineID))
return 0
}
func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("pipelines list", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
jsonOutput := fs.Bool("json", false, "write JSON output")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
ids := sortedPipelineIDs(loaded.Config)
if *jsonOutput {
payload, err := json.Marshal(struct {
Pipelines []string `json:"pipelines"`
}{Pipelines: ids})
if err != nil {
fmt.Fprintf(stderr, "notarius: marshal pipeline list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "%s\n", payload)
return 0
}
for _, id := range ids {
fmt.Fprintln(stdout, id)
}
return 0
}

View File

@@ -0,0 +1,132 @@
package cli
import (
"bytes"
"context"
"errors"
"io"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestConfigKitReadOnlyCommandsLoadImportsAndProfiles(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "development.yml"), "pipelines:\n profiled:\n input: {module: input}\n")
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-profile", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\nprofiled\n" {
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-profile", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
}
}
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "notarius.yml")
writeConfigKitFile(t, path, "version: 4\n")
var stdout, stderr bytes.Buffer
if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") {
t.Fatalf("list = %d, %q", code, stderr.String())
}
}
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, `version: 5
concurrency:
total_llm: 4
pipelines:
"0":
input: {module: input}
`)
writeConfigKitFile(t, filepath.Join(root, "profiles", "development.yml"), "concurrency:\n total_llm: 3\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "retired.yml"), "pipelines: null\n")
providerCalled := false
opts := commandContractOptions(t)
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
providerCalled = true
return nil, nil, errors.New("LLM client must not be constructed")
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-profile", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
if providerCalled {
t.Fatal("config effective constructed an LLM client")
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-profile", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 {
t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-profile", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
}
}
func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
for _, test := range []struct {
name string
pointer string
wantCode int
}{
{name: "missing", wantCode: 2},
{name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2},
{name: "unknown", pointer: "/unknown", wantCode: 1},
{name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
args := []string{"config", "explain", "--config", path}
if test.name != "missing" {
args = append(args, test.pointer)
}
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
t.Fatalf("effective writer failure = %d, %q", code, stderr.String())
}
}
type failingInspectionWriter struct{}
func (failingInspectionWriter) Write([]byte) (int, error) {
return 0, io.ErrClosedPipe
}

View File

@@ -0,0 +1,49 @@
package cli
import (
"errors"
"fmt"
"io"
"os"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
func loadConfigKit(configPath, profile string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
path, err := discoverConfigPath(configPath, opts)
if err != nil {
return nil, "", err
}
workingDirectory, err := os.Getwd()
if err != nil {
return nil, "", fmt.Errorf("determine invocation directory: %w", err)
}
loaded, err := config.Load(path, config.LoadOptions{Profile: profile, Overrides: overrides, InvocationDirectory: workingDirectory})
if err != nil {
return nil, "", err
}
return loaded, path, nil
}
func writeConfigLoadError(output io.Writer, err error) {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
if writeErr := configkit.WriteDiagnostics(output, loadError.Diagnostics()); writeErr == nil {
// ConfigKit's locations contain source IDs. Keep its renderer and
// supply the acquired-source legend so imported/profile errors can
// be traced to a file. Quoting also escapes terminal controls.
for _, source := range loadError.Sources() {
label := source.Path
if label == "" {
label = source.Label
}
if _, err := fmt.Fprintf(output, "source %q: %q\n", source.ID, label); err != nil {
return
}
}
return
}
}
fmt.Fprintf(output, "notarius: %v\n", err)
}

View File

@@ -0,0 +1,85 @@
package cli
import (
"bytes"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
t.Parallel()
for _, profile := range []bool{false, true} {
t.Run(strconv.FormatBool(profile), func(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "config.yml")
fragment := filepath.Join(root, "pipeline\n\x1b.yml")
args := []string{"config", "validate", "--config", main}
if profile {
fragment = filepath.Join(root, "profiles", "development.yml")
writeConfigKitFile(t, main, "version: 5\n")
args = append(args, "--config-profile", "development")
} else {
writeConfigKitFile(t, main, "version: 5\nconfigkit:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n")
}
writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n")
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 {
t.Fatalf("status = %d; %s", code, stderr.String())
}
got := stderr.String()
if stdout.Len() != 0 || !strings.Contains(got, "decode.type") ||
!strings.Contains(got, "source-2:1:") ||
!strings.Contains(got, strconv.Quote(fragment)) ||
strings.Contains(got, "invalid-private-value") ||
strings.ContainsRune(got, '\x1b') {
t.Fatalf("stdout = %q; stderr = %q", stdout.String(), got)
}
})
}
}
func TestLoadConfigKitUsesDiscoveryProfileAndOverrides(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, "version: 5\npipelines:\n base:\n input: {module: input}\n")
profiles := filepath.Join(root, "profiles")
writeConfigKitFile(t, filepath.Join(profiles, "development.yml"), "pipelines:\n profiled:\n input: {module: input}\n")
loaded, gotPath, err := loadConfigKit(path, "development", []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: "generated", Flag: "--output"}}, commandContractOptions(t))
if err != nil {
t.Fatalf("loadConfigKit() error = %v", err)
}
if gotPath != path || loaded.Config.Pipelines["profiled"].Input.Module != "input" || loaded.Config.Output.Directory == "generated" {
t.Fatalf("loaded = %#v, path=%q", loaded.Config, gotPath)
}
}
func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "invalid.yml")
writeConfigKitFile(t, path, "version: 4\n")
_, _, err := loadConfigKit(path, "", nil, commandContractOptions(t))
if err == nil {
t.Fatal("loadConfigKit() error = nil")
}
var output bytes.Buffer
writeConfigLoadError(&output, err)
if !strings.Contains(output.String(), "version") || strings.Contains(output.String(), "notarius: configkit") {
t.Fatalf("diagnostics = %q", output.String())
}
}
func writeConfigKitFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}

View File

@@ -264,13 +264,29 @@ func completeExampleConfigWithTemporaryCache(t *testing.T) string {
updated := strings.Replace(string(content), "directory: ./notarius-cache/chunk-plans", fmt.Sprintf("directory: %q", filepath.Join(cacheRoot, "chunk-plans")), 1) updated := strings.Replace(string(content), "directory: ./notarius-cache/chunk-plans", fmt.Sprintf("directory: %q", filepath.Join(cacheRoot, "chunk-plans")), 1)
updated = strings.Replace(updated, "directory: ./notarius-cache/checkpoints", fmt.Sprintf("directory: %q", filepath.Join(cacheRoot, "checkpoints")), 1) updated = strings.Replace(updated, "directory: ./notarius-cache/checkpoints", fmt.Sprintf("directory: %q", filepath.Join(cacheRoot, "checkpoints")), 1)
for relative, absolute := range map[string]string{ for relative, absolute := range map[string]string{
"./dnd-party.txt": repositoryPath("examples", "dnd-party.txt"), "./llm-profiles/dnd-extraction.yml": repositoryPath("examples", "llm-profiles", "dnd-extraction.yml"),
"./dnd-glossary.txt": repositoryPath("examples", "dnd-glossary.txt"),
"./dnd-spell-catalog.json": repositoryPath("examples", "dnd-spell-catalog.json"),
} { } {
updated = strings.ReplaceAll(updated, relative, fmt.Sprintf("%q", absolute)) updated = strings.ReplaceAll(updated, relative, fmt.Sprintf("%q", absolute))
} }
path := filepath.Join(t.TempDir(), "dnd-complete.config.yml") root := t.TempDir()
fragment, err := os.ReadFile(repositoryPath("examples", "pipelines", "dnd-session.yml"))
if err != nil {
t.Fatal(err)
}
for relative, absolute := range map[string]string{
"../dnd-party.txt": repositoryPath("examples", "dnd-party.txt"),
"../dnd-glossary.txt": repositoryPath("examples", "dnd-glossary.txt"),
"../dnd-spell-catalog.json": repositoryPath("examples", "dnd-spell-catalog.json"),
} {
fragment = []byte(strings.ReplaceAll(string(fragment), relative, fmt.Sprintf("%q", absolute)))
}
if err := os.MkdirAll(filepath.Join(root, "pipelines"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.yml"), fragment, 0o600); err != nil {
t.Fatal(err)
}
path := filepath.Join(root, "dnd-complete.config.yml")
if err := os.WriteFile(path, []byte(updated), 0o600); err != nil { if err := os.WriteFile(path, []byte(updated), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -396,8 +412,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
if err := json.Unmarshal(content, output); err != nil { if err := json.Unmarshal(content, output); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
} }
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, request) client.requests = append(client.requests, snapshot)
client.mu.Unlock() client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
} }
@@ -408,7 +428,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
var requests []contracts.StructuredCompletionRequest var requests []contracts.StructuredCompletionRequest
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if request.PromptID == promptID {
requests = append(requests, request) snapshot, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
} }
} }
return requests return requests

View File

@@ -91,8 +91,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if !reflect.DeepEqual(durable, want) { if !reflect.DeepEqual(durable, want) {
t.Fatalf("durable output payload = %#v, want %#v", durable, want) t.Fatalf("durable output payload = %#v, want %#v", durable, want)
} }
if len(output.Warnings) != 0 { if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings) t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
} }
} }

View File

@@ -147,8 +147,15 @@ func TestMaintainedConfigurationExampleSet(t *testing.T) {
} }
} }
sort.Strings(names) sort.Strings(names)
if got := strings.Join(names, ","); got != "dnd-extraction.yml" { if got := strings.Join(names, ","); got != "development.yml" {
t.Fatalf("maintained operator profiles = %q, want dnd-extraction.yml", got) t.Fatalf("maintained configuration profiles = %q, want development.yml", got)
}
llmProfiles, err := os.ReadDir(repositoryPath("examples", "llm-profiles"))
if err != nil {
t.Fatal(err)
}
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
t.Fatalf("maintained LLM profiles = %#v, want dnd-extraction.yml", llmProfiles)
} }
} }
@@ -239,10 +246,10 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected) t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
} }
warnings := readProductionJSON[struct { warnings := readProductionJSON[struct {
Warnings []json.RawMessage `json:"warnings"` Groups []json.RawMessage `json:"groups"`
}](t, filepath.Join(runRoot, "warnings.json")) }](t, filepath.Join(runRoot, "warnings.json"))
if len(warnings.Warnings) != 0 { if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings) t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
} }
} }

View File

@@ -23,7 +23,7 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test
t.Fatal(err) t.Fatal(err)
} }
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints") checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := fmt.Sprintf(`version: 4 content := fmt.Sprintf(`version: 5
cache: cache:
chunk_plans: chunk_plans:
mode: bypass mode: bypass
@@ -32,14 +32,17 @@ cache:
directory: %q directory: %q
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input:
module: seriatim
artifacts: artifacts:
spells: spells:
extract: extract:
module: dnd/spells module: dnd/spells
references: references:
npc_registry: %q npc_registry:
normalize: dnd/spells path: %q
normalize:
module: dnd/spells
`, checkpointRoot, npcPath) `, checkpointRoot, npcPath)
configPath := filepath.Join(t.TempDir(), "config.yml") configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil { if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {

View File

@@ -19,8 +19,6 @@ import (
"testing/fstest" "testing/fstest"
"time" "time"
"gopkg.in/yaml.v3"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
@@ -333,53 +331,8 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("production prompt engine: %v", err) t.Fatalf("production prompt engine: %v", err)
} }
promptFS, err := components.assets.PromptFS() if engine == nil {
if err != nil { t.Fatal("production PromptKit engine is nil")
t.Fatalf("production prompt assets: %v", err)
}
type manifest struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
Inputs []struct {
Name string `yaml:"name"`
} `yaml:"inputs"`
}
preparedPrompts := 0
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() || filepath.Base(path) != "prompt.yaml" {
return nil
}
data, err := fs.ReadFile(promptFS, path)
if err != nil {
return err
}
var prompt manifest
if err := yaml.Unmarshal(data, &prompt); err != nil {
return err
}
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
for _, input := range prompt.Inputs {
inputs[input.Name] = promptkit.Inline(`{}`)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: prompt.ID, PromptVersion: prompt.Version, ProfileID: "assembled-prompt-test", Inputs: inputs,
})
if err != nil {
return fmt.Errorf("prepare production prompt %q: %w", prompt.ID, err)
}
if prepared.OutputContract.RepairAttempts != 1 {
return fmt.Errorf("prompt %q repair attempts = %d, want 1", prompt.ID, prepared.OutputContract.RepairAttempts)
}
preparedPrompts++
return nil
}); err != nil {
t.Fatal(err)
}
if preparedPrompts == 0 {
t.Fatal("prepared no production prompts")
} }
} }
@@ -729,13 +682,13 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
}{ }{
{ {
name: "unknown module", name: "unknown module",
content: replaceRequiredOnce(t, base, " input: seriatim\n", " input: missing/input\n"), content: replaceRequiredOnce(t, base, " input:\n module: seriatim\n", " input:\n module: missing/input\n"),
options: productionCLIOptions(t), options: productionCLIOptions(t),
fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"}, fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"},
}, },
{ {
name: "unknown validator", name: "unknown validator",
content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n"), content: replaceRequiredOnce(t, base, " extract:\n module: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n"),
options: productionCLIOptions(t), options: productionCLIOptions(t),
fragments: []string{"validator", "missing/validator"}, fragments: []string{"validator", "missing/validator"},
}, },
@@ -747,7 +700,7 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
}, },
{ {
name: "deterministic validator with profile", name: "deterministic validator with profile",
content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n"), content: replaceRequiredOnce(t, base, " extract:\n module: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n"),
options: productionCLIOptions(t), options: productionCLIOptions(t),
fragments: []string{"deterministic validator", "llm_profile"}, fragments: []string{"deterministic validator", "llm_profile"},
}, },
@@ -771,7 +724,7 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) { func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml")) base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n") content := replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
path := writeProductionContractConfig(t, content) path := writeProductionContractConfig(t, content)
components := productionTestComponents(t) components := productionTestComponents(t)
effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
@@ -844,10 +797,10 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations) t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
} }
warnings := readProductionJSON[struct { warnings := readProductionJSON[struct {
Warnings []contracts.Warning `json:"warnings"` Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json")) }](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
if len(warnings.Warnings) != 0 { if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Warnings) t.Fatalf("warnings = %#v, want none", warnings.Groups)
} }
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 { if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts()) t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
@@ -871,7 +824,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
func productionSpellCatalogContractConfig(t *testing.T) string { func productionSpellCatalogContractConfig(t *testing.T) string {
t.Helper() t.Helper()
return fmt.Sprintf(`version: 4 return fmt.Sprintf(`version: 5
cache: cache:
chunk_plans: chunk_plans:
mode: bypass mode: bypass
@@ -880,21 +833,26 @@ cache:
directory: "" directory: ""
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input:
module: seriatim
references: references:
party: %q party:
glossary: %q path: %q
glossary:
path: %q
artifacts: artifacts:
spells: spells:
extract: extract:
module: dnd/spells module: dnd/spells
retries: 2 retries: 2
references: references:
spell_catalog: %q spell_catalog:
path: %q
normalize: normalize:
module: dnd/spells module: dnd/spells
references: references:
spell_catalog: %q spell_catalog:
path: %q
`, repositoryPath("examples", "dnd-party.txt"), repositoryPath("examples", "dnd-glossary.txt"), repositoryPath("examples", "dnd-spell-catalog.json"), repositoryPath("examples", "dnd-spell-catalog.json")) `, repositoryPath("examples", "dnd-party.txt"), repositoryPath("examples", "dnd-glossary.txt"), repositoryPath("examples", "dnd-spell-catalog.json"), repositoryPath("examples", "dnd-spell-catalog.json"))
} }
@@ -905,14 +863,11 @@ func writeProductionSpellCatalogContractConfig(t *testing.T) string {
func loadMaintainedExample(t *testing.T, path string) config.Config { func loadMaintainedExample(t *testing.T, path string) config.Config {
t.Helper() t.Helper()
fileConfig, err := config.LoadFileConfig(path) loaded, err := config.Load(path, config.LoadOptions{})
if err != nil { if err != nil {
t.Fatalf("load maintained config %q: %v", path, err) t.Fatalf("load maintained config %q: %v", path, err)
} }
cfg := config.Default() cfg := loaded.RuntimeConfig()
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
t.Fatalf("apply maintained config %q: %v", path, err)
}
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
t.Fatalf("validate maintained config %q: %v", path, err) t.Fatalf("validate maintained config %q: %v", path, err)
} }
@@ -969,7 +924,7 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
} }
func productionRunConfig(outputRoot, chunkModule string) string { func productionRunConfig(outputRoot, chunkModule string) string {
return fmt.Sprintf(`version: 4 return fmt.Sprintf(`version: 5
output: output:
directory: %q directory: %q
cache: cache:
@@ -980,8 +935,10 @@ debug:
directory: %q directory: %q
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input:
chunk: %s module: seriatim
chunk:
module: %s
output: output:
module: json module: json
options: options:
@@ -990,8 +947,10 @@ pipelines:
- id: identify-items - id: identify-items
artifacts: artifacts:
item-registry: item-registry:
extract: dnd/item-registry extract:
normalize: dnd/item-registry module: dnd/item-registry
normalize:
module: dnd/item-registry
- id: extract-occurrences - id: extract-occurrences
references: references:
item_registry: item_registry:
@@ -1000,9 +959,11 @@ pipelines:
lane: item-registry lane: item-registry
artifacts: artifacts:
spells: spells:
extract: dnd/spells extract:
module: dnd/spells
item-occurrences: item-occurrences:
extract: dnd/item-occurrences extract:
module: dnd/item-occurrences
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) `, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
} }
@@ -1091,8 +1052,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
if err := json.Unmarshal(content, out); err != nil { if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
} }
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, req) client.requests = append(client.requests, snapshot)
client.mu.Unlock() client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
} }
@@ -1103,7 +1068,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
var requests []contracts.StructuredCompletionRequest var requests []contracts.StructuredCompletionRequest
for _, req := range client.requests { for _, req := range client.requests {
if req.PromptID == promptID { if req.PromptID == promptID {
requests = append(requests, req) snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
} }
} }
return requests return requests

View File

@@ -194,7 +194,7 @@ func (recomputeTestOutput) Encode(_ context.Context, req contracts.OutputRequest
func newRecomputeTestRoots(t *testing.T) stateTestRoots { func newRecomputeTestRoots(t *testing.T) stateTestRoots {
t.Helper() t.Helper()
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
config := fmt.Sprintf(`version: 4 config := fmt.Sprintf(`version: 5
output: output:
directory: %q directory: %q
cache: cache:
@@ -208,19 +208,27 @@ debug:
directory: %q directory: %q
pipelines: pipelines:
sample: sample:
input: test/input input:
chunk: test/chunk module: test/input
chunk:
module: test/chunk
steps: steps:
- id: first - id: first
artifacts: artifacts:
producer: producer:
extract: test/extract/producer extract:
merge: test/merge module: test/extract/producer
normalize: test/normalize merge:
module: test/merge
normalize:
module: test/normalize
unrelated: unrelated:
extract: test/extract/unrelated extract:
merge: test/merge module: test/extract/unrelated
normalize: test/normalize merge:
module: test/merge
normalize:
module: test/normalize
- id: second - id: second
references: references:
upstream: upstream:
@@ -229,9 +237,12 @@ pipelines:
lane: producer lane: producer
artifacts: artifacts:
middle: middle:
extract: test/extract/middle extract:
merge: test/merge module: test/extract/middle
normalize: test/normalize merge:
module: test/merge
normalize:
module: test/normalize
- id: third - id: third
references: references:
upstream: upstream:
@@ -240,10 +251,14 @@ pipelines:
lane: middle lane: middle
artifacts: artifacts:
dependent: dependent:
extract: test/extract/dependent extract:
merge: test/merge module: test/extract/dependent
normalize: test/normalize merge:
output: test/recompute-output module: test/merge
normalize:
module: test/normalize
output:
module: test/recompute-output
`, roots.output, roots.plans, roots.checkpoints, roots.debug) `, roots.output, roots.plans, roots.checkpoints, roots.debug)
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil { if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"testing" "testing"
@@ -14,20 +15,16 @@ import (
func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) { func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
selector string selector string
only []string want []string
wantStage pipeline.ModuleStage
wantLane string
wantSlot string
}{ }{
{name: "flat", selector: "alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"}, {name: "pipeline", selector: "shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared", "beta.extract.shared", "beta.merge.shared", "beta.normalize.shared"}},
{name: "chunk", selector: "chunk.chunk-slot", wantStage: pipeline.StageChunk, wantSlot: "chunk-slot"}, {name: "chunk", selector: "chunk.chunk-slot", want: []string{"chunk.chunk-slot"}},
{name: "merge", selector: "merge.alpha-merge", only: []string{"alpha"}, wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"}, {name: "lane", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
{name: "lane", selector: "alpha.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"}, {name: "lane extract", selector: "alpha.extract.alpha-slot", want: []string{"alpha.extract.alpha-slot"}},
{name: "lane extract", selector: "alpha.extract.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"}, {name: "lane merge", selector: "alpha.merge.alpha-merge", want: []string{"alpha.merge.alpha-merge"}},
{name: "lane merge", selector: "alpha.merge.alpha-merge", wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"}, {name: "lane normalize", selector: "alpha.normalize.alpha-normalize", want: []string{"alpha.normalize.alpha-normalize"}},
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", wantStage: pipeline.StageNormalize, wantLane: "alpha", wantSlot: "alpha-normalize"},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -37,70 +34,135 @@ func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil) overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
if err != nil { if err != nil {
t.Fatalf("resolve selector: %v", err) t.Fatalf("resolve selector: %v", err)
} }
if len(overrides) != 1 { if got := referenceContractBindingLabels(overrides); !slices.Equal(got, tt.want) {
t.Fatalf("overrides = %#v, want one binding", overrides) t.Fatalf("binding targets = %#v, want %#v", got, tt.want)
} }
got := overrides[0] for _, binding := range overrides {
if got.Stage != tt.wantStage || got.LaneID != tt.wantLane || got.SlotName != tt.wantSlot || got.BindingSource != contracts.ReferenceBindingSourceCLI { if binding.Source != "reference.txt" || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("binding = %#v, want %s/%s/%s from CLI", got, tt.wantStage, tt.wantLane, tt.wantSlot) t.Fatalf("binding = %#v, want CLI source", binding)
}
})
}
}
func TestReferenceSelectorsRejectAmbiguityWithSpecificSuggestions(t *testing.T) {
cfg := referenceContractConfig()
catalog := referenceContractCatalog(t, true, true)
for _, tt := range []struct {
name string
selector string
want []string
}{
{name: "flat shared slot", selector: "shared", want: []string{"alpha.extract.shared", "beta.extract.shared"}},
{name: "lane shared slot", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
{name: "all mergers", selector: "merge.shared", want: []string{"alpha.merge.shared", "beta.merge.shared"}},
} {
t.Run(tt.name, func(t *testing.T) {
selector, err := parseReferenceSelector(tt.selector, "--reference")
if err != nil {
t.Fatal(err)
}
_, _, err = resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
if err == nil {
t.Fatal("resolve selector succeeded, want ambiguity error")
}
for _, fragment := range tt.want {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("error = %q, want suggestion %q", err, fragment)
} }
} }
}) })
} }
} }
func TestReferenceSelectorsRespectSelectedLanesBeforeMaterialization(t *testing.T) { func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t *testing.T) {
cfg := referenceContractConfig()
catalog := referenceContractCatalog(t, true, true)
requests := []cliReferenceRequest{
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-first.txt"},
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-final.txt"},
{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "lane.txt"},
{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "binding.txt"},
}
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, requests, nil)
if err != nil {
t.Fatal(err)
}
if len(unbinds) != 0 {
t.Fatalf("unbinds = %#v, want none", unbinds)
}
want := map[string]string{
"alpha.extract.shared": "binding.txt",
"alpha.merge.shared": "lane.txt",
"alpha.normalize.shared": "lane.txt",
"beta.extract.shared": "pipeline-final.txt",
"beta.merge.shared": "pipeline-final.txt",
"beta.normalize.shared": "pipeline-final.txt",
}
for _, binding := range overrides {
label := referenceContractBindingLabel(binding)
if binding.Source != want[label] {
t.Fatalf("binding %s source = %q, want %q", label, binding.Source, want[label])
}
delete(want, label)
}
if len(want) != 0 {
t.Fatalf("missing bindings: %#v", want)
}
}
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
catalog := catalogFromRegistries(productionTestComponents(t).registries)
sources := map[string]string{
"party": "/references/party.txt",
"players": "/references/players.txt",
"glossary": "/references/glossary.txt",
"spell_catalog": "/references/spells.json",
}
requests := make([]cliReferenceRequest, 0, len(sources))
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
requests = append(requests, cliReferenceRequest{
Selector: mustParseReferenceSelector(t, slot, "--reference"),
Source: sources[slot],
})
}
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
if err != nil {
t.Fatalf("expand complete D&D references: %v", err)
}
if len(unbinds) != 0 {
t.Fatalf("unbinds = %#v, want none", unbinds)
}
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
for _, binding := range overrides {
actual[referenceContractBindingLabel(binding)] = binding
}
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
if err != nil {
t.Fatal(err)
}
matched := make(map[string]int, len(sources))
for _, target := range targets {
for slot, sourcePath := range sources {
if _, ok := target.slots[slot]; !ok {
continue
}
matched[slot]++
label := targetLabel(target) + "." + slot
binding, ok := actual[label]
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
}
}
}
for slot := range sources {
if matched[slot] < 2 {
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
}
}
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
t.Fatalf("resolve complete D&D CLI references: %v", err)
}
}
func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) {
cfg := referenceContractConfig() cfg := referenceContractConfig()
catalog := referenceContractCatalog(t, true, true) catalog := referenceContractCatalog(t, true, true)
for _, tt := range []struct { for _, tt := range []struct {
name string name string
selector string selector string
only []string
want string want string
}{ }{
{name: "unselected lane", selector: "beta.extract.beta-slot", want: `reference lane "beta" is not selected`}, {name: "pipeline slot", selector: "missing", want: `reference slot "missing" is not declared by any selected target`},
{name: "lane slot", selector: "alpha.missing", want: `reference slot "missing" is not declared by selected lane "alpha"`},
{name: "binding slot", selector: "alpha.extract.missing", want: `reference slot "missing" is not declared`},
{name: "former merge shorthand", selector: "merge.shared", want: `reference lane "merge" is not selected`},
{name: "unselected lane", selector: "beta.extract.beta-slot", only: []string{"alpha"}, want: `reference lane "beta" is not selected`},
{name: "unknown lane", selector: "missing.extract.beta-slot", want: `reference lane "missing" is not selected`}, {name: "unknown lane", selector: "missing.extract.beta-slot", want: `reference lane "missing" is not selected`},
} { } {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
selector, err := parseReferenceSelector(tt.selector, "--reference") selector := mustParseReferenceSelector(t, tt.selector, "--reference")
if err != nil { _, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
t.Fatal(err)
}
_, _, err = resolveCLIReferenceRequests(cfg, "demo", []string{"alpha"}, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
if err == nil || !strings.Contains(err.Error(), tt.want) || strings.Contains(err.Error(), "missing.txt") { if err == nil || !strings.Contains(err.Error(), tt.want) || strings.Contains(err.Error(), "missing.txt") {
t.Fatalf("error = %v, want selection failure before file access", err) t.Fatalf("error = %v, want selection failure containing %q before file access", err, tt.want)
} }
}) })
} }
@@ -131,40 +193,50 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
} }
} }
func TestReferenceOverridesUseFinalExactTargetBinding(t *testing.T) { func TestReferenceBindAndUnbindSpecificity(t *testing.T) {
cfg := referenceContractConfig() cfg := referenceContractConfig()
catalog := referenceContractCatalog(t, true, true) catalog := referenceContractCatalog(t, true, true)
alphaShared, err := parseReferenceSelector("alpha.extract.shared", "--reference") t.Run("specific unbind carves out broad binding", func(t *testing.T) {
if err != nil { overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
t.Fatal(err) []cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "shared.txt"}},
} []cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--without-reference")}},
betaShared, err := parseReferenceSelector("beta.extract.shared", "--reference") )
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{ if got := referenceContractBindingLabels(overrides); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
{Selector: alphaShared, Source: "alpha-first.txt"}, t.Fatalf("overrides = %#v, want all shared targets except alpha extract", got)
{Selector: alphaShared, Source: "alpha-final.txt"}, }
{Selector: betaShared, Source: "beta-only.txt"}, if got := referenceContractUnbindLabels(unbinds); !slices.Equal(got, []string{"alpha.extract.shared"}) {
}, nil) t.Fatalf("unbinds = %#v, want alpha extract", got)
if err != nil { }
t.Fatal(err) })
}
if len(unbinds) != 0 { t.Run("specific binding restores broad unbind", func(t *testing.T) {
t.Fatalf("unbinds = %#v, want none", unbinds) overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
} []cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "alpha.txt"}},
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides}) []cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "shared", "--without-reference")}},
if err != nil { )
t.Fatalf("resolve pipeline: %v", err) if err != nil {
} t.Fatal(err)
alpha := referenceContractLane(t, effective.ResolvedPipeline, "alpha") }
beta := referenceContractLane(t, effective.ResolvedPipeline, "beta") if got := referenceContractBindingLabels(overrides); !slices.Equal(got, []string{"alpha.extract.shared"}) {
if source := referenceContractBindingSource(alpha.ExtractReferences.Bindings, "shared"); source != "alpha-final.txt" { t.Fatalf("overrides = %#v, want alpha extract", got)
t.Fatalf("alpha shared source = %q, want final exact-target override", source) }
} if got := referenceContractUnbindLabels(unbinds); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
if source := referenceContractBindingSource(beta.ExtractReferences.Bindings, "shared"); source != "beta-only.txt" { t.Fatalf("unbinds = %#v, want all shared targets except alpha extract", got)
t.Fatalf("beta shared source = %q, want target-specific override", source) }
} })
t.Run("same specificity conflicts", func(t *testing.T) {
_, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "alpha.txt"}},
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--without-reference")}},
)
if err == nil || !strings.Contains(err.Error(), "same specificity") {
t.Fatalf("error = %v, want same-specificity conflict", err)
}
})
} }
func TestReferenceUnbindsRemoveOptionalAndProtectRequiredSlots(t *testing.T) { func TestReferenceUnbindsRemoveOptionalAndProtectRequiredSlots(t *testing.T) {
@@ -217,7 +289,7 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
workingDir := t.TempDir() workingDir := t.TempDir()
cfg := referenceContractConfig() cfg := referenceContractConfig()
configPath := filepath.Join(configDir, "config.yml") configPath := filepath.Join(configDir, "config.yml")
if err := os.WriteFile(configPath, []byte("version: 4\n"), 0o600); err != nil { if err := os.WriteFile(configPath, []byte("version: 5\n"), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil { if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil {
@@ -257,6 +329,52 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
} }
} }
func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) {
cfg := referenceContractConfig()
profile := cfg.Pipelines["demo"]
alpha := profile.Artifacts["alpha"]
beta := profile.Artifacts["beta"]
alpha.Extract.References["shared"] = pipeline.GeneratedReference("produce", "beta")
profile.Artifacts = nil
profile.Steps = []pipeline.PipelineStepProfile{
{ID: "produce", Artifacts: map[string]pipeline.ArtifactLaneProfile{"beta": beta}},
{ID: "consume", Artifacts: map[string]pipeline.ArtifactLaneProfile{"alpha": alpha}},
}
cfg.Pipelines["demo"] = profile
catalog := referenceContractCatalog(t, true, true)
t.Run("binding conflicts before file access", func(t *testing.T) {
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{
Selector: mustParseReferenceSelector(t, "shared", "--reference"),
Source: filepath.Join(t.TempDir(), "never-read.json"),
}}, nil)
if err != nil {
t.Fatalf("expand CLI reference: %v", err)
}
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") {
t.Fatalf("resolve error = %v, want generated/external conflict before file access", err)
}
})
t.Run("unbind leaves generated source intact", func(t *testing.T) {
_, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, nil, []cliReferenceUnbindRequest{{
Selector: mustParseReferenceSelector(t, "shared", "--without-reference"),
}})
if err != nil {
t.Fatalf("expand CLI unbind: %v", err)
}
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceUnbinds: unbinds})
if err != nil {
t.Fatalf("resolve generated reference with CLI unbind: %v", err)
}
binding := referenceContractFindBinding(referenceContractLane(t, effective.ResolvedPipeline, "alpha").ExtractReferences.Bindings, "shared")
if binding == nil || binding.Artifact == nil || binding.Artifact.Step != "produce" || binding.Artifact.Lane != "beta" {
t.Fatalf("generated binding = %#v, want preserved produce/beta handoff", binding)
}
})
}
func TestReferenceTargetLookupUsesArtifactVariantsAndReportsMissingContext(t *testing.T) { func TestReferenceTargetLookupUsesArtifactVariantsAndReportsMissingContext(t *testing.T) {
cfg := referenceContractConfig() cfg := referenceContractConfig()
full := referenceContractCatalog(t, true, true) full := referenceContractCatalog(t, true, true)
@@ -368,7 +486,7 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil })) register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{})) register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{})) register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil })) register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil })) register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil })) register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
if includeBetaMerger { if includeBetaMerger {
@@ -444,6 +562,42 @@ func referenceContractBindingSource(bindings []pipeline.ReferenceBinding, slot s
return "" return ""
} }
func mustParseReferenceSelector(t *testing.T, value, flagName string) cliReferenceSelector {
t.Helper()
selector, err := parseReferenceSelector(value, flagName)
if err != nil {
t.Fatal(err)
}
return selector
}
func referenceContractBindingLabels(bindings []pipeline.ReferenceBinding) []string {
labels := make([]string, 0, len(bindings))
for _, binding := range bindings {
labels = append(labels, referenceContractBindingLabel(binding))
}
return labels
}
func referenceContractBindingLabel(binding pipeline.ReferenceBinding) string {
if binding.Stage == pipeline.StageChunk {
return "chunk." + binding.SlotName
}
return binding.LaneID + "." + string(binding.Stage) + "." + binding.SlotName
}
func referenceContractUnbindLabels(unbinds []pipeline.ReferenceUnbind) []string {
labels := make([]string, 0, len(unbinds))
for _, unbind := range unbinds {
labels = append(labels, referenceContractBindingLabel(pipeline.ReferenceBinding{
Stage: unbind.Stage,
LaneID: unbind.LaneID,
SlotName: unbind.SlotName,
}))
}
return labels
}
func referenceContractFindBinding(bindings []pipeline.ReferenceBinding, slot string) *pipeline.ReferenceBinding { func referenceContractFindBinding(bindings []pipeline.ReferenceBinding, slot string) *pipeline.ReferenceBinding {
for i := range bindings { for i := range bindings {
if bindings[i].SlotName == slot { if bindings[i].SlotName == slot {

View File

@@ -4,7 +4,6 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
@@ -16,6 +15,7 @@ import (
"strings" "strings"
"time" "time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo" "gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
@@ -34,8 +34,10 @@ const usage = `Usage:
notarius help notarius help
notarius --version notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags] notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius config effective --config path/to/config.yml [--config-profile name]
notarius config explain --config path/to/config.yml [--config-profile name] <json-pointer>
notarius pipelines list --config path/to/config.yml [--config-profile name] [--json]
` `
type Options struct { type Options struct {
@@ -158,6 +160,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fs := flag.NewFlagSet("run", flag.ContinueOnError) fs := flag.NewFlagSet("run", flag.ContinueOnError)
fs.SetOutput(io.Discard) fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path") configPath := fs.String("config", "", "config file path")
configProfile := fs.String("config-profile", "", "configuration profile")
inputPath := fs.String("input", "", "source input file path") inputPath := fs.String("input", "", "source input file path")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes") onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
outputDir := fs.String("output-dir", "", "output directory") outputDir := fs.String("output-dir", "", "output directory")
@@ -177,7 +180,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fs.Var(&llmProfile, "llm-profile", "LLM profile override") fs.Var(&llmProfile, "llm-profile", "LLM profile override")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override") fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh") fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes") fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
if err := validateRunFlagValues(args); err != nil { if err := validateRunFlagValues(args); err != nil {
@@ -265,20 +268,12 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 2 return 2
} }
cfg, loadedConfigPath, err := loadConfig(*configPath, opts) loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configProfile, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
if err != nil { if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) writeConfigLoadError(stderr, err)
return 1 return 1
} }
if chunkCache.set { cfg := loaded.RuntimeConfig()
cfg.Cache.ChunkPlans.Mode = chunkCache.value
}
if dir := strings.TrimSpace(*outputDir); dir != "" {
cfg.Output.Directory = dir
}
if dir := strings.TrimSpace(*debugDir); dir != "" {
cfg.Debug.Directory = dir
}
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1 return 1
@@ -336,6 +331,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
InputPath: strings.TrimSpace(*inputPath), InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath, ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath), ConfigSource: configSource(*configPath),
ConfigProfile: effectiveConfigProfile(*configProfile),
OnlyLanes: append([]string(nil), only...), OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(), ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort, ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
@@ -375,7 +371,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil { if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err)) return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
} }
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath, ConfigPath: loadedConfigPath,
WorkingDir: workingDir, WorkingDir: workingDir,
}) })
@@ -394,7 +390,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
} }
if err := writeSummary(summary, func() error { return summary.WriteRedactedEffectiveConfig(effective) }); err != nil { if err := writeSummary(summary, func() error { return summary.WriteEffectiveConfig(loaded) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err)) return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err))
} }
if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil { if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
@@ -462,7 +458,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt, StartedAt: startedAt,
LLMProfiles: llmProfiles, LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath), Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Warnings: referenceWarnings, Diagnostics: referenceDiagnostics,
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode, ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
ChunkPlans: chunkPlans, ChunkPlans: chunkPlans,
Checkpoints: checkpointRecorder, Checkpoints: checkpointRecorder,
@@ -471,7 +467,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Debug: debugRecorder, Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"], ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
}) })
commandState.observeOutput(output) diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
commandState.observeOutput(output, diagnosticProjection)
}
if err != nil { if err != nil {
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err) primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
@@ -479,15 +478,21 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr)) return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
} }
} }
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr) return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
} }
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
if err := writePartialSummary(summary, output); err != nil { if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err)) return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
} }
var encodedResult []byte var encodedResult []byte
if *machineOutput { if *machineOutput {
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath) result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
if err != nil { if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err) return failPipelineCommand(stderr, commandState, terminalWriter, err)
} }
@@ -513,12 +518,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath) fmt.Fprintf(stdout, "debug=%s\n", debugPath)
} }
} }
if len(output.Warnings) > 0 { if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings)) fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount)
if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok {
fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile))
}
fmt.Fprintln(stderr)
} }
return 0 return 0
} }
func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) {
for _, file := range files {
if file.Name == name {
return file.Name, true
}
}
return "", false
}
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error { func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
if summary == nil { if summary == nil {
return nil return nil
@@ -538,7 +556,7 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
return err return err
} }
} }
if err := summary.WriteWarnings(output.Warnings); err != nil { if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
return err return err
} }
return summary.WriteCheckpointEvents(output.CheckpointEvents) return summary.WriteCheckpointEvents(output.CheckpointEvents)
@@ -859,13 +877,39 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool { func runFlagTakesValue(arg string) bool {
switch arg { switch arg {
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step": case "--config", "--config-profile", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
return true return true
default: default:
return false return false
} }
} }
func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override {
overrides := make([]configkit.Override, 0, 3)
if chunkCache.set {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("cache", "chunk_plans", "mode"),
Value: string(chunkCache.value),
Flag: "--chunk_cache",
})
}
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("output", "directory"),
Value: outputDir,
Flag: "--output-dir",
})
}
if debugDir = strings.TrimSpace(debugDir); debugDir != "" {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("debug", "directory"),
Value: debugDir,
Flag: "--debug-dir",
})
}
return overrides
}
type chunkCacheFlag struct { type chunkCacheFlag struct {
value pipeline.ChunkCacheMode value pipeline.ChunkCacheMode
set bool set bool
@@ -919,7 +963,7 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli
func validateRunFlagValues(args []string) error { func validateRunFlagValues(args []string) error {
for i, arg := range args { for i, arg := range args {
if arg != "--session-id" && arg != "--reasoning-effort" { if arg != "--config-profile" && arg != "--session-id" && arg != "--reasoning-effort" {
continue continue
} }
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
@@ -929,6 +973,13 @@ func validateRunFlagValues(args []string) error {
return nil return nil
} }
func effectiveConfigProfile(profile string) string {
if profile == "" || profile == "default" {
return ""
}
return profile
}
func flagWasProvided(args []string, name string) bool { func flagWasProvided(args []string, name string) bool {
for _, arg := range args { for _, arg := range args {
if arg == name || strings.HasPrefix(arg, name+"=") { if arg == name || strings.HasPrefix(arg, name+"=") {
@@ -1004,6 +1055,10 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
switch args[0] { switch args[0] {
case "validate": case "validate":
return runConfigValidate(args[1:], stdout, stderr, opts) return runConfigValidate(args[1:], stdout, stderr, opts)
case "effective":
return runConfigEffective(args[1:], stdout, stderr, opts)
case "explain":
return runConfigExplain(args[1:], stdout, stderr, opts)
default: default:
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0]) fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
writeUsage(stderr) writeUsage(stderr)
@@ -1011,67 +1066,6 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
} }
} }
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
pipelineID := fs.String("pipeline", "", "pipeline ID")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
if strings.TrimSpace(*onlyRaw) != "" && strings.TrimSpace(*pipelineID) == "" {
fmt.Fprintln(stderr, "notarius: --only requires --pipeline")
return 2
}
only, err := parseOnly(*onlyRaw)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
cfg, path, err := loadConfig(*configPath, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if strings.TrimSpace(*pipelineID) != "" {
catalog, err := effectiveCatalog(opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: *pipelineID,
Only: only,
Catalog: catalog,
})
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "config %q is valid for pipeline %q\n", path, strings.TrimSpace(*pipelineID))
return 0
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "config %q is valid\n", path)
return 0
}
func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int { func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
if len(args) == 0 { if len(args) == 0 {
fmt.Fprintln(stderr, "notarius: pipelines requires a subcommand") fmt.Fprintln(stderr, "notarius: pipelines requires a subcommand")
@@ -1088,70 +1082,6 @@ func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
} }
} }
func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("pipelines list", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
jsonOutput := fs.Bool("json", false, "write JSON output")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
cfg, _, err := loadConfig(*configPath, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
ids := sortedPipelineIDs(cfg)
if *jsonOutput {
payload := struct {
Pipelines []string `json:"pipelines"`
}{Pipelines: ids}
encoded, err := json.Marshal(payload)
if err != nil {
fmt.Fprintf(stderr, "notarius: marshal pipeline list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "%s\n", encoded)
return 0
}
for _, id := range ids {
fmt.Fprintln(stdout, id)
}
return 0
}
func loadConfig(configPath string, opts Options) (config.Config, string, error) {
path, err := discoverConfigPath(configPath, opts)
if err != nil {
return config.Config{}, "", err
}
fileCfg, err := config.LoadFileConfig(path)
if err != nil {
return config.Config{}, "", err
}
cfg := config.Default()
if err := cfg.ApplyFileConfigWithLookup(fileCfg, opts.LookupEnv); err != nil {
return config.Config{}, "", err
}
if err := cfg.ApplyEnvOverridesWithLookup(opts.LookupEnv); err != nil {
return config.Config{}, "", err
}
return cfg, path, nil
}
func discoverConfigPath(configPath string, opts Options) (string, error) { func discoverConfigPath(configPath string, opts Options) (string, error) {
if path := strings.TrimSpace(configPath); path != "" { if path := strings.TrimSpace(configPath); path != "" {
if err := requireConfigFile(path); err != nil { if err := requireConfigFile(path); err != nil {
@@ -1265,11 +1195,21 @@ type cliReferenceUnbindRequest struct {
} }
type cliReferenceSelector struct { type cliReferenceSelector struct {
Scope cliReferenceSelectorScope
LaneID string LaneID string
Stage pipeline.ModuleStage Stage pipeline.ModuleStage
SlotName string SlotName string
} }
type cliReferenceSelectorScope uint8
const (
cliReferenceScopePipeline cliReferenceSelectorScope = iota
cliReferenceScopeLane
cliReferenceScopeChunk
cliReferenceScopeBinding
)
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) { func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if len(values) == 0 { if len(values) == 0 {
return nil, nil return nil, nil
@@ -1278,7 +1218,7 @@ func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
for _, raw := range values { for _, raw := range values {
name, source, ok := strings.Cut(raw, "=") name, source, ok := strings.Cut(raw, "=")
if !ok { if !ok {
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path") return nil, fmt.Errorf("--reference must use slot=path, lane.slot=path, or lane.stage.slot=path")
} }
if strings.TrimSpace(source) == "" { if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind") return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
@@ -1328,17 +1268,14 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
} }
switch len(parts) { switch len(parts) {
case 1: case 1:
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil return cliReferenceSelector{Scope: cliReferenceScopePipeline, SlotName: strings.TrimSpace(parts[0])}, nil
case 2: case 2:
first := strings.TrimSpace(parts[0]) first := strings.TrimSpace(parts[0])
slotName := strings.TrimSpace(parts[1]) slotName := strings.TrimSpace(parts[1])
if first == string(pipeline.StageChunk) { if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil return cliReferenceSelector{Scope: cliReferenceScopeChunk, Stage: pipeline.StageChunk, SlotName: slotName}, nil
} }
if first == string(pipeline.StageMerge) { return cliReferenceSelector{Scope: cliReferenceScopeLane, LaneID: first, SlotName: slotName}, nil
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3: case 3:
laneID := strings.TrimSpace(parts[0]) laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1])) stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
@@ -1346,9 +1283,9 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize { if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
} }
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil return cliReferenceSelector{Scope: cliReferenceScopeBinding, LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default: default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
} }
} }
@@ -1369,37 +1306,153 @@ func resolveCLIReferenceRequests(
return nil, nil, err return nil, nil, err
} }
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests)) // Broad CLI selectors are only presentation syntax. Collapse them into one
// highest-specificity action per concrete framework target before pipeline
// resolution so the generic reference contract stays stage-and-lane exact.
actions := make(map[cliReferenceTargetKey]resolvedCLIReferenceAction)
for _, request := range referenceRequests { for _, request := range referenceRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector) matches, err := resolveCLIReferenceTargets(targets, request.Selector)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
overrides = append(overrides, pipeline.ReferenceBinding{ for _, target := range matches {
Stage: target.stage, candidate := resolvedCLIReferenceAction{
LaneID: target.laneID, kind: cliReferenceActionBind,
SlotName: request.Selector.SlotName, selector: request.Selector,
Source: request.Source, target: target,
BindingSource: contracts.ReferenceBindingSourceCLI, slotName: request.Selector.SlotName,
}) source: request.Source,
specificity: request.Selector.specificity(),
}
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
return nil, nil, err
}
}
} }
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests { for _, request := range unbindRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector) matches, err := resolveCLIReferenceTargets(targets, request.Selector)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
unbinds = append(unbinds, pipeline.ReferenceUnbind{ for _, target := range matches {
Stage: target.stage, candidate := resolvedCLIReferenceAction{
LaneID: target.laneID, kind: cliReferenceActionUnbind,
SlotName: request.Selector.SlotName, selector: request.Selector,
}) target: target,
slotName: request.Selector.SlotName,
specificity: request.Selector.specificity(),
}
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
return nil, nil, err
}
}
} }
resolved := make([]resolvedCLIReferenceAction, 0, len(actions))
for _, action := range actions {
resolved = append(resolved, action)
}
sort.Slice(resolved, func(i, j int) bool {
left, right := resolved[i], resolved[j]
if left.target.laneID != right.target.laneID {
return left.target.laneID < right.target.laneID
}
if left.target.stage != right.target.stage {
return referenceStageOrder(left.target.stage) < referenceStageOrder(right.target.stage)
}
return left.slotName < right.slotName
})
overrides := make([]pipeline.ReferenceBinding, 0, len(resolved))
unbinds := make([]pipeline.ReferenceUnbind, 0, len(resolved))
for _, action := range resolved {
switch action.kind {
case cliReferenceActionBind:
overrides = append(overrides, pipeline.ReferenceBinding{
Stage: action.target.stage,
LaneID: action.target.laneID,
SlotName: action.slotName,
Source: action.source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
case cliReferenceActionUnbind:
unbinds = append(unbinds, pipeline.ReferenceUnbind{
Stage: action.target.stage,
LaneID: action.target.laneID,
SlotName: action.slotName,
})
}
}
return overrides, unbinds, nil return overrides, unbinds, nil
} }
type cliReferenceActionKind uint8
const (
cliReferenceActionBind cliReferenceActionKind = iota
cliReferenceActionUnbind
)
type cliReferenceTargetKey struct {
stage pipeline.ModuleStage
laneID string
slotName string
}
type resolvedCLIReferenceAction struct {
kind cliReferenceActionKind
selector cliReferenceSelector
target selectedReferenceTarget
slotName string
source string
specificity int
}
func mergeCLIReferenceAction(actions map[cliReferenceTargetKey]resolvedCLIReferenceAction, candidate resolvedCLIReferenceAction) error {
key := cliReferenceTargetKey{stage: candidate.target.stage, laneID: candidate.target.laneID, slotName: candidate.slotName}
current, ok := actions[key]
if !ok || candidate.specificity > current.specificity {
actions[key] = candidate
return nil
}
if candidate.specificity < current.specificity {
return nil
}
if candidate.kind != current.kind {
return fmt.Errorf("reference target %q slot %q is both bound by %q and unbound by %q at the same specificity", targetLabel(candidate.target), candidate.slotName, current.selector.String(), candidate.selector.String())
}
actions[key] = candidate
return nil
}
func (selector cliReferenceSelector) specificity() int {
switch selector.Scope {
case cliReferenceScopePipeline:
return 0
case cliReferenceScopeLane:
return 1
case cliReferenceScopeChunk, cliReferenceScopeBinding:
return 2
default:
return -1
}
}
func (selector cliReferenceSelector) String() string {
switch selector.Scope {
case cliReferenceScopePipeline:
return selector.SlotName
case cliReferenceScopeLane:
return selector.LaneID + "." + selector.SlotName
case cliReferenceScopeChunk:
return "chunk." + selector.SlotName
case cliReferenceScopeBinding:
return selector.LaneID + "." + string(selector.Stage) + "." + selector.SlotName
default:
return selector.SlotName
}
}
type selectedReferenceTarget struct { type selectedReferenceTarget struct {
laneID string laneID string
stage pipeline.ModuleStage stage pipeline.ModuleStage
@@ -1616,114 +1669,77 @@ func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
return slotSet return slotSet
} }
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) { func resolveCLIReferenceTargets(targets []selectedReferenceTarget, selector cliReferenceSelector) ([]selectedReferenceTarget, error) {
slotName := strings.TrimSpace(selector.SlotName) slotName := strings.TrimSpace(selector.SlotName)
if slotName == "" { if slotName == "" {
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty") return nil, fmt.Errorf("reference slot must not be empty")
} }
if selector.Stage == pipeline.StageChunk { switch selector.Scope {
case cliReferenceScopePipeline:
matches := make([]selectedReferenceTarget, 0, len(targets))
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if len(matches) == 0 {
return nil, fmt.Errorf("reference slot %q is not declared by any selected target", slotName)
}
return matches, nil
case cliReferenceScopeChunk:
for _, target := range targets { for _, target := range targets {
if target.stage != pipeline.StageChunk { if target.stage != pipeline.StageChunk {
continue continue
} }
if _, ok := target.slots[slotName]; !ok { if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module) return nil, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
} }
return target, nil return []selectedReferenceTarget{target}, nil
}
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
}
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
} }
return nil, fmt.Errorf("reference chunk target is not selected")
case cliReferenceScopeLane:
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 3)
for _, target := range targets { for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage { if target.laneID != selector.LaneID {
if _, ok := target.slots[slotName]; !ok { continue
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target)) }
} laneSelected = true
return target, nil if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
} }
} }
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID) if !laneSelected {
} return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
if strings.TrimSpace(selector.LaneID) != "" {
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
}
return resolveCLIReferenceFlatTarget(targets, slotName)
}
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.stage != stage {
continue
} }
if _, ok := target.slots[slotName]; ok { if len(matches) == 0 {
matches = append(matches, target) return nil, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, selector.LaneID)
} }
} return matches, nil
switch len(matches) { case cliReferenceScopeBinding:
case 0: laneSelected := false
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage) for _, target := range targets {
case 1: if target.laneID != selector.LaneID {
return matches[0], nil continue
}
laneSelected = true
if target.stage != selector.Stage {
continue
}
if _, ok := target.slots[slotName]; !ok {
return nil, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
}
return []selectedReferenceTarget{target}, nil
}
if !laneSelected {
return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
}
return nil, fmt.Errorf("reference %s target is not selected for lane %q", selector.Stage, selector.LaneID)
default: default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName)) return nil, fmt.Errorf("reference selector has unknown scope")
} }
} }
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.laneID != laneID {
continue
}
laneSelected = true
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if !laneSelected {
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func targetList(targets []selectedReferenceTarget) string {
labels := make([]string, 0, len(targets))
for _, target := range targets {
labels = append(labels, targetLabel(target))
}
sort.Strings(labels)
return strings.Join(labels, ", ")
}
func targetLabel(target selectedReferenceTarget) string { func targetLabel(target selectedReferenceTarget) string {
if target.stage == pipeline.StageChunk { if target.stage == pipeline.StageChunk {
return "chunk" return "chunk"
@@ -1731,17 +1747,19 @@ func targetLabel(target selectedReferenceTarget) string {
return target.laneID + "." + string(target.stage) return target.laneID + "." + string(target.stage)
} }
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string { func referenceStageOrder(stage pipeline.ModuleStage) int {
suggestions := make([]string, 0, len(targets)) switch stage {
for _, target := range targets { case pipeline.StageChunk:
if target.stage == pipeline.StageChunk { return 0
suggestions = append(suggestions, "chunk."+slotName) case pipeline.StageExtract:
continue return 1
} case pipeline.StageMerge:
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName) return 2
case pipeline.StageNormalize:
return 3
default:
return 4
} }
sort.Strings(suggestions)
return strings.Join(suggestions, " or ")
} }
func sortedPipelineIDs(cfg config.Config) []string { func sortedPipelineIDs(cfg config.Config) []string {

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -37,6 +38,9 @@ func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
{name: "blank debug directory", args: func(roots stateTestRoots) []string { {name: "blank debug directory", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""} return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
}}, }},
{name: "missing configuration profile", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-profile"}
}},
{name: "debug directory without debug", args: func(roots stateTestRoots) []string { {name: "debug directory without debug", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")} return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
}}, }},
@@ -79,7 +83,7 @@ func TestRecomputeStepCLIContract(t *testing.T) {
{ {
name: "explicit step", name: "explicit step",
configure: func(t *testing.T, roots stateTestRoots) { configure: func(t *testing.T, roots stateTestRoots) {
replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n") replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n")
}, },
flags: []string{"--resume", "--recompute-step", "chosen"}, flags: []string{"--resume", "--recompute-step", "chosen"},
wantCode: 0, wantCode: 0,
@@ -161,7 +165,7 @@ func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
data = []byte(replaceRequiredOnce(t, string(data), " output: test/output\n", " other:\n extract: test/extract\n output: test/output\n")) data = []byte(replaceRequiredOnce(t, string(data), " output:\n module: test/output\n", " other:\n extract:\n module: test/extract\n output:\n module: test/output\n"))
if err := os.WriteFile(roots.config, data, 0o600); err != nil { if err := os.WriteFile(roots.config, data, 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -180,7 +184,7 @@ func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
} }
func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) { func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
t.Run("environment roots", func(t *testing.T) { t.Run("retired environment roots have no effect", func(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
environmentOutput := filepath.Join(t.TempDir(), "environment-output") environmentOutput := filepath.Join(t.TempDir(), "environment-output")
environmentDebug := filepath.Join(t.TempDir(), "environment-debug") environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
@@ -193,10 +197,10 @@ func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
if result.code != 0 { if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
} }
assertFile(t, filepath.Join(environmentOutput, filepath.Base(onlyChildDir(t, environmentOutput)), "result.json")) assertFile(t, filepath.Join(roots.output, filepath.Base(onlyChildDir(t, roots.output)), "result.json"))
onlyChildDir(t, environmentDebug) onlyChildDir(t, roots.debug)
assertAbsent(t, roots.output) assertAbsent(t, environmentOutput)
assertAbsent(t, roots.debug) assertAbsent(t, environmentDebug)
}) })
t.Run("command flags override environment", func(t *testing.T) { t.Run("command flags override environment", func(t *testing.T) {
@@ -227,23 +231,57 @@ func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
text := string(data) text := string(data)
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.output), "") text = replaceRequiredOnce(t, text, fmt.Sprintf("output:\n directory: %q\n", roots.output), "")
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.debug), "") text = replaceRequiredOnce(t, text, fmt.Sprintf("debug:\n directory: %q\n", roots.debug), "")
if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil { if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
workDir := t.TempDir()
t.Chdir(workDir)
opts := newStateTestHarness().options() opts := newStateTestHarness().options()
result := runWithStateRoots(t, roots, opts, nil) result := runWithStateRoots(t, roots, opts, nil)
if result.code != 0 { if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
} }
assertFile(t, filepath.Join(workDir, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(workDir, "notarius-output"))), "result.json")) configDirectory := filepath.Dir(roots.config)
onlyChildDir(t, filepath.Join(workDir, "notarius-debug")) assertFile(t, filepath.Join(configDirectory, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(configDirectory, "notarius-output"))), "result.json"))
onlyChildDir(t, filepath.Join(configDirectory, "notarius-debug"))
}) })
} }
func TestRunAppliesConfigurationProfileAndRecordsSelection(t *testing.T) {
roots := newStateTestRoots(t)
profileDirectory := filepath.Join(filepath.Dir(roots.config), "profiles")
profileOutput := filepath.Join(profileDirectory, "profile-output")
profileDebug := filepath.Join(profileDirectory, "profile-debug")
writeConfigKitFile(t, filepath.Join(profileDirectory, "development.yml"), "output:\n directory: ./profile-output\ndebug:\n directory: ./profile-debug\n")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--config-profile", "development",
"--input", roots.input, "--chunk_cache", "bypass", "--debug",
}, &stdout, &stderr, newStateTestHarness().options())
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertFile(t, filepath.Join(profileOutput, filepath.Base(onlyChildDir(t, profileOutput)), "result.json"))
bundle := onlyChildDir(t, profileDebug)
var invocation debugbundle.Invocation
readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation)
if invocation.ConfigProfile != "development" {
t.Fatalf("config profile = %q, want development", invocation.ConfigProfile)
}
}
func TestEffectiveConfigProfileOmitsBaseSelection(t *testing.T) {
for _, profile := range []string{"", "default"} {
if got := effectiveConfigProfile(profile); got != "" {
t.Fatalf("effectiveConfigProfile(%q) = %q, want empty", profile, got)
}
}
if got := effectiveConfigProfile("development"); got != "development" {
t.Fatalf("effectiveConfigProfile(development) = %q", got)
}
}
func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) { t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
@@ -567,7 +605,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
data = []byte(replaceRequiredOnce(t, string(data), "extract: test/extract", "extract: test/failing-extract")) data = []byte(replaceRequiredOnce(t, string(data), "module: test/extract", "module: test/failing-extract"))
if err := os.WriteFile(roots.config, data, 0o600); err != nil { if err := os.WriteFile(roots.config, data, 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -590,10 +628,11 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
harness := newStateTestHarness() harness := newStateTestHarness()
harness.includeWarnings = true harness.includeWarnings = true
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}} harness.includeWarningFile = true
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options()) code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") { if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
} }
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json") outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
@@ -601,11 +640,12 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
if err != nil || !strings.Contains(string(output), "contract-warning") { if err != nil || !strings.Contains(string(output), "contract-warning") {
t.Fatalf("durable output = %q, %v", output, err) t.Fatalf("durable output = %q, %v", output, err)
} }
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
bundle := onlyChildDir(t, roots.debug) bundle := onlyChildDir(t, roots.debug)
var warnings []contracts.Warning var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings) readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" { if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
t.Fatalf("debug warnings = %#v", warnings) t.Fatalf("debug diagnostics = %#v", diagnostics)
} }
} }

View File

@@ -7,25 +7,40 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
const runResultSchemaVersion = "notarius.run-result.v1" const runResultSchemaVersion = "notarius.run-result.v2"
type runResult struct { type runResult struct {
SchemaVersion string `json:"schema_version"` SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"` RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"` PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"` OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"` IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"` NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"` RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"` WarningGroupCount int `json:"warning_group_count"`
ValidationStatus string `json:"validation_status"` WarningOccurrenceCount int `json:"warning_occurrence_count"`
DebugDirectory string `json:"debug_directory,omitempty"` DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
} }
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) { func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if err != nil {
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
}
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
}
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
if strings.TrimSpace(output.Manifest.RunID) == "" { if strings.TrimSpace(output.Manifest.RunID) == "" {
return runResult{}, fmt.Errorf("run result requires a run ID") return runResult{}, fmt.Errorf("run result requires a run ID")
} }
@@ -51,14 +66,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
} }
result := runResult{ result := runResult{
SchemaVersion: runResultSchemaVersion, SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID, RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID, PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory, OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs), NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected), RejectedOutputCount: len(output.Rejected),
WarningCount: len(output.Warnings), WarningGroupCount: len(diagnosticProjection.Warnings),
ValidationStatus: output.Manifest.ValidationStatus, WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
} }
if strings.TrimSpace(debugDirectory) != "" { if strings.TrimSpace(debugDirectory) != "" {
@@ -85,6 +105,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
return result, nil return result, nil
} }
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
}
func encodeRunResult(result runResult) ([]byte, error) { func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result) encoded, err := json.Marshal(result)
if err != nil { if err != nil {

View File

@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
} }
receipt := decodeRunResultDocument(t, stdout.String()) receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v1" { if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got) t.Fatalf("schema_version = %q", got)
} }
if got := receipt["run_id"]; got != productionRunID { if got := receipt["run_id"]; got != productionRunID {
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
if got := receipt["rejected_output_count"]; got != float64(0) { if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got) t.Fatalf("rejected_output_count = %v", got)
} }
if got := receipt["warning_count"]; got != float64(0) { if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", receipt)
} }
if got := receipt["validation_status"]; got != "approved" { if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got) t.Fatalf("validation_status = %q", got)
@@ -63,19 +63,19 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) { func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
harness := newStateTestHarness() harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}} harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{ code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input, "run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json", "--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options()) }, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") { if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
} }
receipt := decodeRunResultDocument(t, stdout.String()) receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_count"]; got != float64(1) { if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", receipt)
} }
debugDirectory, ok := receipt["debug_directory"].(string) debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n")) configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil { if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -52,8 +52,11 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
if got := decoded["rejected_output_count"]; got != float64(1) { if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got) t.Fatalf("rejected_output_count = %v", got)
} }
if got := decoded["warning_count"]; got != float64(1) { if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", decoded)
}
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
} }
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") { if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got) t.Fatalf("output_directory = %q", got)
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
} }
} }
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
output := testRunOutput()
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
if err != nil {
t.Fatal(err)
}
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
t.Fatalf("encoded result = %s", encoded)
}
}
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) { func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -159,8 +182,15 @@ func testRunOutput() pipeline.RunOutput {
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"}, Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}}, NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}}, Rejected: []contracts.RejectedOutput{{}},
Warnings: []contracts.Warning{{}}, Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
OutputFiles: []contracts.OutputFile{{Name: "index.json"}}, Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryFallback,
ReasonCode: "fallback",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
} }
} }

View File

@@ -6,6 +6,7 @@ import (
"io" "io"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -33,13 +34,17 @@ func (s *pipelineCommandState) setDebugPath(debugPath string) {
} }
} }
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) { func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput, diagnostics contracts.DiagnosticProjection) {
if s == nil { if s == nil {
return return
} }
s.report.OutputCount = len(output.NormalizeOutputs) s.report.OutputCount = len(output.NormalizeOutputs)
s.report.RejectedCount = len(output.Rejected) s.report.RejectedCount = len(output.Rejected)
s.report.WarningCount = len(output.Warnings) s.report.WarningGroupCount = len(diagnostics.Warnings)
s.report.WarningOccurrenceCount = diagnostics.WarningOccurrenceCount
s.report.DiagnosticGroupCount = len(diagnostics.Diagnostics)
s.report.DiagnosticOccurrenceCount = diagnostics.DiagnosticOccurrenceCount
s.report.DiagnosticsTruncated = output.Diagnostics.Truncated
s.report.ValidationStatus = output.Manifest.ValidationStatus s.report.ValidationStatus = output.Manifest.ValidationStatus
} }

View File

@@ -238,7 +238,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
}, },
} }
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil { if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -264,7 +264,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision) t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
} }
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3") changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil { if err != nil {

View File

@@ -19,12 +19,12 @@ import (
func TestProductionSpellCatalogValidationRetries(t *testing.T) { func TestProductionSpellCatalogValidationRetries(t *testing.T) {
const retries = 2 const retries = 2
tests := []struct { tests := []struct {
name string name string
responses []string responses []string
wantCalls int wantCalls int
wantRejected bool wantRejected bool
wantSpell string wantSpell string
wantWarningCode string wantAdvisoryCode string
}{ }{
{ {
name: "unknown spell remains rejected after exhaustion", name: "unknown spell remains rejected after exhaustion",
@@ -42,9 +42,9 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
productionSpellResponse("Unknown Spell"), productionSpellResponse("Unknown Spell"),
productionSpellResponse("Aegis of Emberfall"), productionSpellResponse("Aegis of Emberfall"),
}, },
wantCalls: 2, wantCalls: 2,
wantSpell: "Aegis of Emberfall", wantSpell: "Aegis of Emberfall",
wantWarningCode: "spell_not_near_source", wantAdvisoryCode: "spell_not_near_source",
}, },
} }
@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
t.Fatalf("materialize production references: %v", err) t.Fatalf("materialize production references: %v", err)
} }
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
llmClient := &catalogRetryLLMClient{responses: tt.responses} llmClient := &catalogRetryLLMClient{responses: tt.responses}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient}) prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 { if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection) t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
} }
if len(output.Warnings) != 0 { if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want no emitted warnings from rejected attempts", output.Warnings) t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
} }
return return
} }
@@ -107,8 +108,16 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell { if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value) t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
} }
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode { if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings) t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
}
if len(output.Diagnostics.Groups) != 2 {
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
}
for _, diagnostic := range output.Diagnostics.Groups {
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
}
} }
}) })
} }

View File

@@ -57,7 +57,6 @@ func TestRunStateSurfaceMatrix(t *testing.T) {
} }
} }
assertAnyFile(t, roots.checkpoints) assertAnyFile(t, roots.checkpoints)
assertRestrictedTree(t, roots.checkpoints)
if debug { if debug {
bundle := onlyChildDir(t, roots.debug) bundle := onlyChildDir(t, roots.debug)
assertFile(t, filepath.Join(bundle, "summary", "invocation.json")) assertFile(t, filepath.Join(bundle, "summary", "invocation.json"))
@@ -259,7 +258,7 @@ func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testi
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
configText := replaceRequiredOnce(t, string(data), " input: test/input\n", ` input: configText := replaceRequiredOnce(t, string(data), " input:\n module: test/input\n", ` input:
module: test/input module: test/input
options: options:
api_key: CONFIG_SUMMARY_SECRET_SENTINEL api_key: CONFIG_SUMMARY_SECRET_SENTINEL
@@ -277,7 +276,7 @@ func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testi
t.Fatalf("code=%d stderr=%q", result.code, result.stderr) t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
} }
summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary") summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary")
for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} { for _, name := range []string{"effective-config.yml", "resolved-pipeline.json"} {
contents, err := os.ReadFile(filepath.Join(summaryRoot, name)) contents, err := os.ReadFile(filepath.Join(summaryRoot, name))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -349,7 +348,7 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID) t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID)
} }
report := readStateTestRunReport(t, debugPath) report := readStateTestRunReport(t, debugPath)
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" { if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "approved" {
t.Fatalf("success report = %#v", report) t.Fatalf("success report = %#v", report)
} }
if !strings.Contains(result.stdout, "outputs=1 rejected=0") { if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
@@ -431,7 +430,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
bundlePath := onlyChildDir(t, roots.debug) bundlePath := onlyChildDir(t, roots.debug)
runID := filepath.Base(bundlePath) runID := filepath.Base(bundlePath)
report := readStateTestRunReport(t, bundlePath) report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation { if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != tc.wantValidation {
t.Fatalf("failure report = %#v", report) t.Fatalf("failure report = %#v", report)
} }
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
@@ -445,7 +444,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) { func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
roots := newStateTestRoots(t) roots := newStateTestRoots(t)
harness := newStateTestHarness() harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}} harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")}
harness.extractErr = errors.New("synthetic partial pipeline failure") harness.extractErr = errors.New("synthetic partial pipeline failure")
result := runStateTest(t, roots, harness.options(), true, true, "bypass") result := runStateTest(t, roots, harness.options(), true, true, "bypass")
@@ -454,7 +453,7 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
} }
bundlePath := onlyChildDir(t, roots.debug) bundlePath := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, bundlePath) report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" { if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningGroupCount != 1 || report.WarningOccurrenceCount != 1 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "failed" {
t.Fatalf("partial failure report = %#v", report) t.Fatalf("partial failure report = %#v", report)
} }
@@ -463,10 +462,10 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" { if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
t.Fatalf("partial manifest = %#v", manifest) t.Fatalf("partial manifest = %#v", manifest)
} }
var warnings []contracts.Warning var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings) readStateTestSummaryJSON(t, bundlePath, "final-diagnostics.json", &diagnostics)
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" { if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" {
t.Fatalf("partial warnings = %#v", warnings) t.Fatalf("partial diagnostics = %#v", diagnostics)
} }
var events []pipeline.CheckpointEvent var events []pipeline.CheckpointEvent
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events) readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
@@ -688,7 +687,7 @@ func newStateTestRoots(t *testing.T) stateTestRoots {
t.Fatal(err) t.Fatal(err)
} }
roots.config = filepath.Join(base, "config.yml") roots.config = filepath.Join(base, "config.yml")
config := fmt.Sprintf("version: 4\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug) config := fmt.Sprintf("version: 5\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input:\n module: test/input\n chunk:\n module: test/chunk\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n output:\n module: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug)
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil { if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -863,11 +862,12 @@ type stateTestHarness struct {
chunkCalls, extractCalls int chunkCalls, extractCalls int
runIDCalls uint64 runIDCalls uint64
extractErr error extractErr error
chunkWarnings []contracts.Warning chunkDiagnostics []contracts.ProducerDiagnostic
moduleProfiles []string moduleProfiles []string
sessionIDs []string sessionIDs []string
outputWarnings []contracts.Warning outputDiagnostics contracts.DiagnosticCollection
includeWarnings bool includeWarnings bool
includeWarningFile bool
} }
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} } func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
@@ -892,7 +892,7 @@ func (h *stateTestHarness) options() Options {
panic(err) panic(err)
} }
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil
}); err != nil { }); err != nil {
panic(err) panic(err)
} }
@@ -931,7 +931,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
c.harness.mu.Lock() c.harness.mu.Lock()
c.harness.chunkCalls++ c.harness.chunkCalls++
c.harness.mu.Unlock() c.harness.mu.Unlock()
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil
} }
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact" const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
@@ -999,20 +999,28 @@ func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNor
} }
type stateTestOutput struct { type stateTestOutput struct {
harness *stateTestHarness harness *stateTestHarness
includeWarnings bool includeDiagnostics bool
} }
func (o stateTestOutput) Key() string { return "test/output" } func (o stateTestOutput) Key() string { return "test/output" }
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
o.harness.mu.Lock() o.harness.mu.Lock()
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...) o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics)
o.harness.mu.Unlock() o.harness.mu.Unlock()
data := []byte("{\"ok\":true}\n") data := []byte("{\"ok\":true}\n")
if o.includeWarnings && len(req.Warnings) > 0 { if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode)) data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
} }
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil files := []contracts.OutputFile{{Name: "result.json", Bytes: data}}
if o.harness.includeWarningFile {
files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")})
}
return contracts.OutputResult{Files: files}, nil
}
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {
return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}}
} }
type failingDebugRecorder struct{} type failingDebugRecorder struct{}

View File

@@ -0,0 +1,53 @@
//go:build linux || darwin
package cli
import (
"fmt"
"os"
"path/filepath"
"syscall"
"testing"
)
func TestRunCreatesReusableGroupAccessibleCacheState(t *testing.T) {
previousUmask := syscall.Umask(0o007)
defer syscall.Umask(previousUmask)
roots := newStateTestRoots(t)
harness := newStateTestHarness()
result := runStateTest(t, roots, harness.options(), false, false, "auto")
if result.code != 0 {
t.Fatalf("initial run code=%d stderr=%q", result.code, result.stderr)
}
assertGroupAccessibleCacheTree(t, roots.plans)
assertGroupAccessibleCacheTree(t, roots.checkpoints)
extractCalls := harness.extractCalls
result = runStateTest(t, roots, harness.options(), false, true, "auto")
if result.code != 0 {
t.Fatalf("resume run code=%d stderr=%q", result.code, result.stderr)
}
if harness.extractCalls != extractCalls {
t.Fatalf("extract calls after reuse = %d, want %d", harness.extractCalls, extractCalls)
}
}
func assertGroupAccessibleCacheTree(t *testing.T, root string) {
t.Helper()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
want := os.FileMode(0o660)
if info.IsDir() {
want = 0o770
}
if got := info.Mode().Perm(); got != want {
return fmt.Errorf("%s mode = %#o, want %#o", path, got, want)
}
return nil
}); err != nil {
t.Fatal(err)
}
}

View File

@@ -93,17 +93,44 @@ type NormalizedOutputManifest struct {
} }
type RejectedOutputManifest struct { type RejectedOutputManifest struct {
Stage string `json:"stage"` Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"` StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"` LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"` ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"` ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"` ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"` ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"` ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
AttemptCount int `json:"attempt_count,omitempty"` AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Validation *ValidationSummary `json:"validation,omitempty"`
}
// ValidationSummary is the bounded, durable outcome of validating one
// producer result. It deliberately contains identities and stable codes, not
// model responses, corrective guidance, validator diagnostics, or payloads.
type ValidationSummary struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Status string `json:"status"`
RejectingValidators []string `json:"rejecting_validators,omitempty"`
ReasonCodes []string `json:"reason_codes,omitempty"`
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
ProducerAttemptCount int `json:"producer_attempt_count"`
TerminalAction string `json:"terminal_action"`
}
// CloneValidationSummary returns an independently owned durable summary.
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
return summary
} }
type CheckpointDecisionManifest struct { type CheckpointDecisionManifest struct {
@@ -163,6 +190,7 @@ type RunManifest struct {
References []ReferenceProvenance `json:"references,omitempty"` References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"` NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"` RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"` CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`

View File

@@ -4,8 +4,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
const SupportedFileConfigVersion = 4
type Config struct { type Config struct {
PromptKit PromptKitConfig `json:"promptkit,omitempty"` PromptKit PromptKitConfig `json:"promptkit,omitempty"`
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"` Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
@@ -29,9 +27,6 @@ type PromptKitLocalBackendConfig struct {
type ConcurrencyConfig struct { type ConcurrencyConfig struct {
TotalLLM int `json:"total_llm"` TotalLLM int `json:"total_llm"`
StageWorkers map[string]int `json:"stage_workers"` StageWorkers map[string]int `json:"stage_workers"`
extractWorkersConfigured bool
defaultedExtractWorkers int
} }
const defaultLLMConcurrency = 16 const defaultLLMConcurrency = 16
@@ -62,9 +57,8 @@ func Default() Config {
return Config{ return Config{
Pipelines: map[string]pipeline.PipelineProfile{}, Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{ Concurrency: ConcurrencyConfig{
TotalLLM: defaultLLMConcurrency, TotalLLM: defaultLLMConcurrency,
StageWorkers: map[string]int{"extract": defaultLLMConcurrency}, StageWorkers: map[string]int{"extract": defaultLLMConcurrency},
defaultedExtractWorkers: defaultLLMConcurrency,
}, },
Output: OutputConfig{Directory: "./notarius-output"}, Output: OutputConfig{Directory: "./notarius-output"},
Cache: CacheConfig{ChunkPlans: ChunkPlanCacheConfig{Mode: pipeline.ChunkCacheAuto}}, Cache: CacheConfig{ChunkPlans: ChunkPlanCacheConfig{Mode: pipeline.ChunkCacheAuto}},
@@ -97,25 +91,20 @@ func cloneIntMap(in map[string]int) map[string]int {
return out return out
} }
func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() { func (c Config) withDerivedStageWorkers() Config {
if c == nil { c = cloneConfig(c)
return if _, exists := c.Concurrency.StageWorkers["extract"]; !exists {
} if c.Concurrency.StageWorkers == nil {
if c.StageWorkers == nil { c.Concurrency.StageWorkers = make(map[string]int)
c.StageWorkers = make(map[string]int)
}
if !c.extractWorkersConfigured {
if value, ok := c.StageWorkers["extract"]; ok && (c.defaultedExtractWorkers == 0 || value != c.defaultedExtractWorkers) {
c.extractWorkersConfigured = true
return
} }
c.StageWorkers["extract"] = c.TotalLLM c.Concurrency.StageWorkers["extract"] = c.Concurrency.TotalLLM
c.defaultedExtractWorkers = c.TotalLLM
} }
return c
} }
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile { func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil { if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value out.StructuredOutputRepairAttempts = &value
@@ -200,6 +189,7 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil { if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value out.StructuredOutputRepairAttempts = &value
@@ -212,6 +202,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out return out
} }
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride { func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set} out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 { if len(in.Validators) > 0 {

View File

@@ -0,0 +1,122 @@
package config
import "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
// Document is the version 5 configuration shape decoded by ConfigKit. It is
// deliberately separate from Config so YAML presence and composition concerns
// do not reach runtime pipeline types.
type Document struct {
Version string `yaml:"version"`
PromptKit *PromptKitDocument `yaml:"promptkit,omitempty"`
Pipelines map[string]PipelineDocument `yaml:"pipelines,omitempty"`
Concurrency *ConcurrencyDocument `yaml:"concurrency,omitempty"`
Output *OutputDocument `yaml:"output,omitempty"`
Cache *CacheDocument `yaml:"cache,omitempty"`
Debug *DebugDocument `yaml:"debug,omitempty"`
}
type PromptKitDocument struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
LocalBackend *PromptKitLocalBackendDocument `yaml:"local_backend,omitempty"`
}
type PromptKitLocalBackendDocument struct {
Endpoint *string `yaml:"endpoint,omitempty"`
ConcurrencyLimit *int `yaml:"concurrency_limit,omitempty"`
}
type PipelineDocument struct {
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
Input BindingDocument `yaml:"input"`
Chunk *BindingDocument `yaml:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts,omitempty"`
Steps []PipelineStepDocument `yaml:"steps,omitempty"`
Output *BindingDocument `yaml:"output,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
}
type PipelineStepDocument struct {
ID string `yaml:"id"`
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
}
type ArtifactLaneDocument struct {
Extract BindingDocument `yaml:"extract"`
Merge *BindingDocument `yaml:"merge,omitempty"`
Normalize *BindingDocument `yaml:"normalize,omitempty"`
Validators []ValidatorDocument `yaml:"validators,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
}
// BindingDocument represents a producer binding. ValidatorDocument is a
// deliberately finite subset so nested validator-only fields are rejected by
// ConfigKit's strict decoder.
type BindingDocument struct {
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
Retries int `yaml:"retries,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
Validators []ValidatorDocument `yaml:"validators,omitempty"`
}
type ValidatorDocument struct {
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
Retries int `yaml:"retries,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
}
// ReferenceDocument is the explicit version 5 union. Later document
// validation establishes which locations permit generated references and
// requires exactly one populated branch.
type ReferenceDocument struct {
Path *string `yaml:"path,omitempty"`
Artifact *ArtifactReferenceDocument `yaml:"artifact,omitempty"`
}
type ArtifactReferenceDocument struct {
Step string `yaml:"step"`
Lane string `yaml:"lane"`
}
type ValidationPolicyDocument struct {
ProducerStructuralFailure *pipeline.ProducerStructuralFailureAction `yaml:"producer_structural_failure,omitempty"`
SemanticRejection *pipeline.SemanticRejectionAction `yaml:"semantic_rejection,omitempty"`
ValidatorFailure *pipeline.ValidatorFailureAction `yaml:"validator_failure,omitempty"`
}
type ConcurrencyDocument struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
}
type OutputDocument struct {
Directory *string `yaml:"directory,omitempty"`
}
type CacheDocument struct {
ChunkPlans *ChunkPlanCacheDocument `yaml:"chunk_plans,omitempty"`
Checkpoints *CheckpointCacheDocument `yaml:"checkpoints,omitempty"`
}
type ChunkPlanCacheDocument struct {
Directory *string `yaml:"directory,omitempty"`
Mode *string `yaml:"mode,omitempty"`
}
type CheckpointCacheDocument struct {
Enabled *bool `yaml:"enabled,omitempty"`
Directory *string `yaml:"directory,omitempty"`
}
type DebugDocument struct {
Directory *string `yaml:"directory,omitempty"`
}

View File

@@ -0,0 +1,26 @@
package config
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
// documentNormalizer records leaf provenance and, while visiting a sequence,
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
// values, so a record for only a descendant is insufficient. The outermost
// sequence covers nested sequences as well; unchanged visits record nothing.
type documentNormalizer struct {
*configkit.Resolution
sequence *configkit.Path
}
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
if n.sequence == nil {
n.sequence = &path
}
return n
}
func (n documentNormalizer) Record(path configkit.Path, kind configkit.ChangeKind, reason string, dependencies ...configkit.Path) {
n.Resolution.Record(path, kind, reason, dependencies...)
if n.sequence != nil {
n.Resolution.Record(*n.sequence, kind, reason, append([]configkit.Path{path}, dependencies...)...)
}
}

View File

@@ -0,0 +1,167 @@
package config
import (
"path/filepath"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
resolution := documentNormalizer{Resolution: context}
if document == nil {
return
}
initializeDocumentMainDirectory(resolution, state)
if document.PromptKit != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("promptkit", "profile_dir"), &document.PromptKit.ProfileDir, false, "resolved PromptKit profile directory")
normalizeDocumentPath(resolution, state, configkit.Keys("promptkit", "profile_file"), &document.PromptKit.ProfileFile, false, "resolved PromptKit profile file")
}
if document.Output != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("output", "directory"), &document.Output.Directory, false, "resolved output directory")
}
if document.Cache != nil {
if document.Cache.ChunkPlans != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("cache", "chunk_plans", "directory"), &document.Cache.ChunkPlans.Directory, true, "resolved chunk-plan cache directory")
}
if document.Cache.Checkpoints != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("cache", "checkpoints", "directory"), &document.Cache.Checkpoints.Directory, true, "resolved checkpoint cache directory")
}
}
if document.Debug != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("debug", "directory"), &document.Debug.Directory, false, "resolved debug directory")
}
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
path := configkit.Keys("pipelines").Key(id)
normalizeDocumentReferencePaths(profile.References, path.Key("references"), resolution, state)
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil {
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), resolution, state)
}
if profile.Output != nil {
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state)
}
for laneID, lane := range profile.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
profile.Artifacts[laneID] = lane
}
stepResolution := resolution.inSequence(path.Key("steps"))
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
for laneID, lane := range step.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
step.Artifacts[laneID] = lane
}
}
document.Pipelines[id] = profile
}
}
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
if state == nil || state.mainDirectory != "" {
return
}
info, ok := resolution.InputInfo(configkit.Keys("version"))
if ok && len(info.Origin.Locations) == 1 {
if source, sourceOK := resolution.Source(info.Origin.Locations[0].SourceID); sourceOK && source.Path != "" {
state.mainDirectory = filepath.Dir(source.Path)
return
}
}
// Byte and reader sources are test-oriented private entry points. They have
// no declaring file, so preserve their established invocation-directory
// semantics without extending that fallback to file-backed production loads.
state.mainDirectory = cleanInvocationDirectory(".")
}
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
if binding != nil {
normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
}
}
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
if lane == nil {
return
}
normalizeDocumentBindingReferencePaths(&lane.Extract, path.Key("extract"), resolution, state)
normalizeDocumentBindingReferencePaths(lane.Merge, path.Key("merge"), resolution, state)
normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
}
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name).Key("path")
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
base, ok := documentPathBase(resolution, state, sourcePath)
if !ok {
resolution.Error(sourcePath, "relative path has no unambiguous source base")
} else {
value := filepath.Clean(filepath.Join(base, *source.Path))
if value != *source.Path {
resolution.Record(sourcePath, configkit.ChangeNormalization, "resolved external reference path")
source.Path = documentValue(value)
}
}
references[name] = source
}
}
}
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
return
}
base, ok := documentPathBase(resolution, state, path)
if !ok {
resolution.Error(path, "relative path has no unambiguous source base")
return
}
value := filepath.Clean(filepath.Join(base, **target))
if optional && **target == "" {
return
}
if *target != nil && **target != value {
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
}
}
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
original := path
if state != nil {
original = state.originalPath(path)
}
info, ok := resolution.InputInfo(original)
if !ok {
if state != nil && state.mainDirectory != "" {
return state.mainDirectory, true
}
return "", false
}
switch info.Origin.Layer {
case configkit.LayerCLI:
if state != nil && state.invocationDirectory != "" {
return state.invocationDirectory, true
}
case configkit.LayerDefault:
if state != nil && state.mainDirectory != "" {
return state.mainDirectory, true
}
default:
if len(info.Origin.Locations) != 1 {
return "", false
}
source, ok := resolution.Source(info.Origin.Locations[0].SourceID)
if !ok || source.Path == "" {
if state != nil && state.mainDirectory != "" {
return state.mainDirectory, true
}
return "", false
}
return filepath.Dir(source.Path), true
}
return "", false
}

View File

@@ -0,0 +1,123 @@
package config
import (
"os"
"path/filepath"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
t.Parallel()
root := t.TempDir()
main := filepath.Join(root, "notarius.yml")
fragmentDirectory := filepath.Join(root, "fragments")
if err := os.MkdirAll(fragmentDirectory, 0o700); err != nil {
t.Fatal(err)
}
fragment := filepath.Join(fragmentDirectory, "pipelines.yml")
if err := os.WriteFile(fragment, []byte(`pipelines:
imported:
input: {module: input}
references:
source: {path: ./from-fragment.txt}
`), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(main, []byte(`version: 5
configkit:
imports:
- path: fragments/pipelines.yml
output: {directory: ./output}
debug: {directory: ./debug}
pipelines:
main:
input: {module: input}
references:
source: {path: ./from-main.txt}
`), 0o600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(main, LoadOptions{})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if got := loaded.Config.Pipelines["main"].References["source"].Path; got != filepath.Join(root, "from-main.txt") {
t.Fatalf("main reference = %q", got)
}
if got := loaded.Config.Pipelines["imported"].References["source"].Path; got != filepath.Join(fragmentDirectory, "from-fragment.txt") {
t.Fatalf("fragment reference = %q", got)
}
if loaded.Config.Output.Directory != filepath.Join(root, "output") || loaded.Config.Debug.Directory != filepath.Join(root, "debug") {
t.Fatalf("destinations = %#v %#v", loaded.Config.Output, loaded.Config.Debug)
}
}
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
t.Parallel()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(configPath, []byte("version: 5\n"), 0o600); err != nil {
t.Fatal(err)
}
invocation := t.TempDir()
loaded, err := loadConfig(configPath, LoadOptions{
InvocationDirectory: invocation,
Overrides: []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: "generated", Flag: "--output"}},
})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if got := loaded.Config.Output.Directory; got != filepath.Join(invocation, "generated") {
t.Fatalf("override directory = %q", got)
}
}
func TestDocumentPathsUseProfileDeclarationDirectory(t *testing.T) {
t.Parallel()
root := t.TempDir()
profiles := filepath.Join(root, "profiles")
if err := os.MkdirAll(profiles, 0o700); err != nil {
t.Fatal(err)
}
main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte("version: 5\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "development.yml"), []byte("output: {directory: ./profile-output}\n"), 0o600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(main, LoadOptions{Profile: "development"})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if got := loaded.Config.Output.Directory; got != filepath.Join(profiles, "profile-output") {
t.Fatalf("profile directory = %q", got)
}
}
func TestDocumentPathsKeepEmptyCacheDirectoriesAndAbsolutePaths(t *testing.T) {
t.Parallel()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
abs := filepath.Join(t.TempDir(), "outside.txt")
if err := os.WriteFile(configPath, []byte("version: 5\ncache:\n chunk_plans: {directory: ''}\n checkpoints: {directory: ''}\npipelines:\n main:\n input: {module: input}\n references:\n outside: {path: "+abs+"}\n"), 0o600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(configPath, LoadOptions{})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if loaded.Config.Cache.ChunkPlans.Directory != "" || loaded.Config.Cache.Checkpoints.Directory != "" {
t.Fatalf("cache directories = %#v", loaded.Config.Cache)
}
if got := loaded.Config.Pipelines["main"].References["outside"].Path; got != abs {
t.Fatalf("absolute reference = %q", got)
}
}

View File

@@ -0,0 +1,154 @@
package config
import (
"errors"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestDocumentLoadAcceptsVersion5AndCanonicalShapes(t *testing.T) {
t.Parallel()
for _, source := range []string{
`version: 5
pipelines:
session:
input:
module: seriatim
references:
party:
path: ./party.txt
steps:
- id: describe
artifacts:
spells:
extract:
module: dnd/spells
validators:
- module: generic/valid_json
options:
strict: true
- id: use-artifact
references:
spells:
artifact:
step: describe
lane: spells
artifacts:
locations:
extract:
module: dnd/locations
`,
`version: "5"
pipelines:
session:
input:
module: seriatim
`,
} {
loaded, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(source)), LoadOptions{})
if err != nil {
t.Fatalf("loadDocumentSource() error = %v", err)
}
profile, ok := loaded.Config.Pipelines["session"]
if !ok || profile.Input.Module != "seriatim" {
t.Fatalf("pipeline = %#v", profile)
}
}
}
func TestDocumentLoadRejectsUnsupportedVersionsBeforeDocumentDecoding(t *testing.T) {
t.Parallel()
tests := []struct {
name string
source string
code configkit.Code
}{
{name: "missing", source: "unknown: value\n", code: configkit.CodeVersionError},
{name: "null", source: "version: null\nunknown: value\n", code: configkit.CodeVersionError},
{name: "version four", source: "version: 4\nunknown: value\n", code: configkit.CodeVersionError},
{name: "unsupported", source: "version: 6\nunknown: value\n", code: configkit.CodeVersionError},
{name: "noninteger", source: "version: 5.0\n", code: configkit.CodeVersionType},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(test.source)), LoadOptions{})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %v, want ConfigKit load error", err)
}
diagnostics := loadError.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != test.code || diagnostics[0].Path.String() != "/version" {
t.Fatalf("diagnostics = %#v", diagnostics)
}
if strings.Contains(diagnostics[0].Message, "unknown") {
t.Fatalf("document decoding ran before version validation: %#v", diagnostics)
}
})
}
}
func TestDocumentLoadRejectsShorthandAndUnknownFields(t *testing.T) {
t.Parallel()
tests := []struct {
name string
source string
code configkit.Code
path string
}{
{
name: "binding shorthand",
source: "version: 5\npipelines:\n session:\n input: seriatim\n",
code: configkit.CodeDecodeType,
path: "/pipelines/session/input",
},
{
name: "validator nested producer field",
source: "version: 5\npipelines:\n session:\n input:\n module: seriatim\n validators:\n - module: generic/valid_json\n validators: []\n",
code: configkit.CodeDecodeUnknownField,
path: "/pipelines/session/input/validators/0/validators",
},
{
name: "unknown field",
source: "version: 5\npipelines:\n session:\n input:\n module: seriatim\n unknown: true\n",
code: configkit.CodeDecodeUnknownField,
path: "/pipelines/session/input/unknown",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(test.source)), LoadOptions{})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %v, want ConfigKit load error", err)
}
diagnostics := loadError.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != test.code || diagnostics[0].Path.String() != test.path {
t.Fatalf("diagnostics = %#v", diagnostics)
}
})
}
}
func TestDocumentDefinitionReturnsIndependentDefaults(t *testing.T) {
t.Parallel()
first, err := loadDocumentSource(configkit.Bytes("first.yml", []byte("version: 5\n")), LoadOptions{})
if err != nil {
t.Fatal(err)
}
first.Config.Pipelines["changed"] = PipelineDocument{}
first.Config.Concurrency.StageWorkers["extract"] = 3
second, err := loadDocumentSource(configkit.Bytes("second.yml", []byte("version: 5\n")), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if len(second.Config.Pipelines) != 0 || second.Config.Concurrency.StageWorkers["extract"] != defaultLLMConcurrency || len(second.Config.Concurrency.StageWorkers) != 1 {
t.Fatalf("defaults aliased across loads: %#v", second.Config)
}
}

View File

@@ -26,7 +26,7 @@ type EffectiveConfig struct {
} }
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) { func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
c.Concurrency.recomputeStageWorkerDefaults() c = c.withDerivedStageWorkers()
if err := c.Validate(); err != nil { if err := c.Validate(); err != nil {
return EffectiveConfig{}, err return EffectiveConfig{}, err
} }

View File

@@ -1,103 +0,0 @@
package config
import (
"fmt"
"os"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func LoadFromEnv() (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverridesWithLookup(os.LookupEnv); err != nil {
return Config{}, err
}
return cfg, nil
}
func (c *Config) ApplyEnvOverrides() error {
return c.applyEnvOverridesWithLookup(os.LookupEnv)
}
func (c *Config) ApplyEnvOverridesWithLookup(lookup func(string) (string, bool)) error {
return c.applyEnvOverridesWithLookup(lookup)
}
func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok {
value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw)
if err != nil {
return err
}
c.Concurrency.TotalLLM = value
}
if raw, ok := lookup("NOTARIUS_STAGE_WORKERS_EXTRACT"); ok {
value, err := parseIntEnv("NOTARIUS_STAGE_WORKERS_EXTRACT", raw)
if err != nil {
return err
}
if c.Concurrency.StageWorkers == nil {
c.Concurrency.StageWorkers = make(map[string]int)
}
c.Concurrency.StageWorkers["extract"] = value
c.Concurrency.extractWorkersConfigured = true
}
c.Concurrency.recomputeStageWorkerDefaults()
if raw, ok := lookup("NOTARIUS_OUTPUT_DIR"); ok {
c.Output.Directory = strings.TrimSpace(raw)
if c.Output.Directory == "" {
return fmt.Errorf("NOTARIUS_OUTPUT_DIR: must not be empty")
}
if strings.ContainsRune(c.Output.Directory, '\x00') {
return fmt.Errorf("NOTARIUS_OUTPUT_DIR: must not contain NUL")
}
}
if raw, ok := lookup("NOTARIUS_CACHE_CHUNK_PLANS_MODE"); ok {
mode, err := pipeline.ParseChunkCacheMode(raw)
if err != nil {
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_MODE: %w", err)
}
c.Cache.ChunkPlans.Mode = mode
}
if raw, ok := lookup("NOTARIUS_CACHE_CHUNK_PLANS_DIR"); ok {
c.Cache.ChunkPlans.Directory = cleanOptionalPath(raw)
if c.Cache.ChunkPlans.Directory == "" {
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_DIR: must not be empty")
}
if strings.ContainsRune(c.Cache.ChunkPlans.Directory, '\x00') {
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_DIR: must not contain NUL")
}
}
if raw, ok := lookup("NOTARIUS_CACHE_CHECKPOINTS_DIR"); ok {
c.Cache.Checkpoints.Directory = cleanOptionalPath(raw)
if c.Cache.Checkpoints.Directory == "" {
return fmt.Errorf("NOTARIUS_CACHE_CHECKPOINTS_DIR: must not be empty")
}
if strings.ContainsRune(c.Cache.Checkpoints.Directory, '\x00') {
return fmt.Errorf("NOTARIUS_CACHE_CHECKPOINTS_DIR: must not contain NUL")
}
}
if raw, ok := lookup("NOTARIUS_DEBUG_DIR"); ok {
c.Debug.Directory = strings.TrimSpace(raw)
if c.Debug.Directory == "" {
return fmt.Errorf("NOTARIUS_DEBUG_DIR: must not be empty")
}
if strings.ContainsRune(c.Debug.Directory, '\x00') {
return fmt.Errorf("NOTARIUS_DEBUG_DIR: must not contain NUL")
}
}
return nil
}
func parseIntEnv(name string, raw string) (int, error) {
value, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil {
return 0, fmt.Errorf("%s: must be an integer", name)
}
return value, nil
}

View File

@@ -1,239 +0,0 @@
package config
import (
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestPrecedenceFileValuesOverrideBuiltInDefaults(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
concurrency:
total_llm: 4
stage_workers:
extract: 2
output:
directory: ./file-output
cache:
chunk_plans:
directory: ./file-plans
mode: refresh
checkpoints:
directory: ./file-checkpoints
debug:
directory: ./file-debug
`)
if cfg.Concurrency.TotalLLM != 4 || cfg.Concurrency.StageWorkers["extract"] != 2 ||
cfg.Output.Directory != "./file-output" || cfg.Cache.ChunkPlans.Directory != "file-plans" ||
cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheRefresh || cfg.Cache.Checkpoints.Directory != "file-checkpoints" ||
cfg.Debug.Directory != "./file-debug" {
t.Fatalf("file values did not override defaults: %#v", cfg)
}
}
func TestPrecedenceOperationalEnvironmentOverridesFileValues(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
concurrency:
total_llm: 2
stage_workers:
extract: 1
output:
directory: ./file-output
cache:
chunk_plans:
directory: ./file-plans
mode: refresh
checkpoints:
directory: ./file-checkpoints
debug:
directory: ./file-debug
`)
env := map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "8",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "6",
"NOTARIUS_OUTPUT_DIR": "/env/output",
"NOTARIUS_CACHE_CHUNK_PLANS_MODE": "bypass",
"NOTARIUS_CACHE_CHUNK_PLANS_DIR": "/env/plans",
"NOTARIUS_CACHE_CHECKPOINTS_DIR": "/env/checkpoints",
"NOTARIUS_DEBUG_DIR": "/env/debug",
}
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(env)); err != nil {
t.Fatal(err)
}
if cfg.Concurrency.TotalLLM != 8 || cfg.Concurrency.StageWorkers["extract"] != 6 ||
cfg.Output.Directory != "/env/output" || cfg.Cache.ChunkPlans.Directory != "/env/plans" ||
cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass || cfg.Cache.Checkpoints.Directory != "/env/checkpoints" ||
cfg.Debug.Directory != "/env/debug" {
t.Fatalf("environment values did not override file values: %#v", cfg)
}
}
func TestPrecedenceExtractWorkersFollowEffectiveConcurrencyUnlessExplicit(t *testing.T) {
tests := []struct {
name string
file string
env map[string]string
wantTotal int
wantWorker int
}{
{
name: "default follows environment total",
file: "version: 4\n",
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5"},
wantTotal: 5,
wantWorker: 5,
},
{
name: "file worker is retained",
file: "version: 4\nconcurrency:\n total_llm: 3\n stage_workers:\n extract: 2\n",
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"},
wantTotal: 6,
wantWorker: 2,
},
{
name: "environment worker is retained",
file: "version: 4\nconcurrency:\n total_llm: 2\n",
env: map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "4",
},
wantTotal: 6,
wantWorker: 4,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := applyFileConfig(t, tt.file)
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(tt.env)); err != nil {
t.Fatal(err)
}
if cfg.Concurrency.TotalLLM != tt.wantTotal || cfg.Concurrency.StageWorkers["extract"] != tt.wantWorker {
t.Fatalf("concurrency = %#v, want total %d and extract %d", cfg.Concurrency, tt.wantTotal, tt.wantWorker)
}
})
}
}
func TestPrecedenceEmptyFileCacheDirectoriesDeferPerUserResolution(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
cache:
chunk_plans:
directory: ""
checkpoints:
directory: ""
`)
if err := cfg.Validate(); err != nil {
t.Fatalf("empty file cache directories should be valid: %v", err)
}
if cfg.Cache.ChunkPlans.Directory != "" || cfg.Cache.Checkpoints.Directory != "" {
t.Fatalf("empty cache directories were not preserved for deferred resolution: %#v", cfg.Cache)
}
resolver := func() (string, error) { return "/user/cache", nil }
chunkPlans, err := DefaultChunkPlanRoot(resolver)
if err != nil {
t.Fatal(err)
}
checkpoints, err := DefaultCheckpointRoot(resolver)
if err != nil {
t.Fatal(err)
}
if chunkPlans != "/user/cache/notarius/chunk-plans" || checkpoints != "/user/cache/notarius/checkpoints" {
t.Fatalf("deferred cache roots = %q, %q", chunkPlans, checkpoints)
}
}
func TestDefaultCacheRootsRejectInvalidUserCacheResolvers(t *testing.T) {
tests := []struct {
name string
resolver func() (string, error)
want string
}{
{name: "nil resolver", want: "must not be nil"},
{
name: "resolver failure",
resolver: func() (string, error) {
return "", errors.New("cache home unavailable")
},
want: "resolve user cache directory",
},
{name: "empty directory", resolver: func() (string, error) { return " ", nil }, want: "must not be empty"},
}
families := []struct {
name string
root func(func() (string, error)) (string, error)
}{
{name: "chunk plans", root: DefaultChunkPlanRoot},
{name: "checkpoints", root: DefaultCheckpointRoot},
}
for _, family := range families {
for _, tt := range tests {
t.Run(family.name+"/"+tt.name, func(t *testing.T) {
_, err := family.root(tt.resolver)
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %v, want substring %q", err, tt.want)
}
})
}
}
}
func TestEnvEmptyDirectoryOverridesAreErrors(t *testing.T) {
tests := []string{
"NOTARIUS_OUTPUT_DIR",
"NOTARIUS_CACHE_CHUNK_PLANS_DIR",
"NOTARIUS_CACHE_CHECKPOINTS_DIR",
"NOTARIUS_DEBUG_DIR",
}
for _, name := range tests {
t.Run(name, func(t *testing.T) {
cfg := Default()
err := cfg.ApplyEnvOverridesWithLookup(lookupValues(map[string]string{name: " \t"}))
if err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("error = %v, want responsible environment variable", err)
}
})
}
}
func TestEnvInvalidIntegersAndChunkCacheModesReportTheirNames(t *testing.T) {
tests := map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "not-an-integer",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "not-an-integer",
"NOTARIUS_CACHE_CHUNK_PLANS_MODE": "not-a-cache-mode",
}
for name, value := range tests {
t.Run(name, func(t *testing.T) {
cfg := Default()
err := cfg.ApplyEnvOverridesWithLookup(lookupValues(map[string]string{name: value}))
if err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("error = %v, want responsible environment variable", err)
}
})
}
}
func TestEnvRemovedProviderVariablesAreIgnored(t *testing.T) {
before := Default()
cfg := Default()
removed := map[string]string{
"NOTARIUS_LLM_DEFAULT_ENDPOINT": "ignored-provider-setting",
"NOTARIUS_LLM_DEFAULT_MODEL": "ignored-provider-setting",
}
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(removed)); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(cfg, before) {
t.Fatalf("removed provider variables changed configuration: %#v", cfg)
}
}
func lookupValues(values map[string]string) func(string) (string, bool) {
return func(name string) (string, bool) {
value, ok := values[name]
return value, ok
}
}

View File

@@ -1,835 +0,0 @@
package config
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gopkg.in/yaml.v3"
)
type FileConfig struct {
Version int `yaml:"version"`
PromptKit *FilePromptKitConfig `yaml:"promptkit,omitempty"`
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
Output *FileOutputConfig `yaml:"output,omitempty"`
Cache *FileCacheConfig `yaml:"cache,omitempty"`
Debug *FileDebugConfig `yaml:"debug,omitempty"`
}
type FilePromptKitConfig struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
LocalBackend *FilePromptKitLocalBackendConfig `yaml:"local_backend,omitempty"`
}
type FilePromptKitLocalBackendConfig struct {
Endpoint *string `yaml:"endpoint,omitempty"`
ConcurrencyLimit *int `yaml:"concurrency_limit,omitempty"`
}
type FilePipelineProfile struct {
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
llmProfileSet bool `yaml:"-"`
}
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
if err := validateStructuredOutputRepairAttemptsNode(node, "pipeline profile"); err != nil {
return err
}
type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile")
if err != nil {
return err
}
*p = FilePipelineProfile(decoded)
_, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"]
return nil
}
func (s *FilePipelineStepProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineStepProfile FilePipelineStepProfile
var decoded plainFilePipelineStepProfile
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"id": {}, "artifacts": {}, "references": {},
}, "pipeline step"); err != nil {
return err
}
*s = FilePipelineStepProfile(decoded)
return nil
}
func (l *FileArtifactLaneProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFileArtifactLaneProfile FileArtifactLaneProfile
var decoded plainFileArtifactLaneProfile
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"extract": {}, "merge": {}, "normalize": {}, "validators": {}, "references": {},
}, "artifact lane"); err != nil {
return err
}
*l = FileArtifactLaneProfile(decoded)
return nil
}
func decodeKnownMapping(node *yaml.Node, target any, allowed map[string]struct{}, context string) (map[string]struct{}, error) {
if node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s must be an object", context)
}
if err := node.Decode(target); err != nil {
return nil, err
}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s field %q is duplicated", context, key)
}
if _, ok := allowed[key]; !ok {
return nil, fmt.Errorf("field %s not found in %s", key, context)
}
seen[key] = struct{}{}
}
return seen, nil
}
type FilePipelineStepProfile struct {
ID string `yaml:"id"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
}
type FileArtifactLaneProfile struct {
Extract fileModuleBinding `yaml:"extract"`
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
}
type FileConcurrencyConfig struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
}
type FileOutputConfig struct {
Directory *string `yaml:"directory,omitempty"`
}
type FileCacheConfig struct {
ChunkPlans *FileChunkPlanCacheConfig `yaml:"chunk_plans,omitempty"`
Checkpoints *FileCheckpointCacheConfig `yaml:"checkpoints,omitempty"`
}
type FileChunkPlanCacheConfig struct {
Directory *string `yaml:"directory,omitempty"`
Mode *string `yaml:"mode,omitempty"`
}
type FileCheckpointCacheConfig struct {
Enabled *bool `yaml:"enabled,omitempty"`
Directory *string `yaml:"directory,omitempty"`
}
type FileDebugConfig struct {
Directory *string `yaml:"directory,omitempty"`
}
type fileModuleBinding struct {
Module string
LLMProfile string
StructuredOutputRepairAttempts *int
Retries int
Options map[string]any
References map[string]fileReferenceSource
Validators pipeline.ValidatorOverride
}
type fileReferenceSource struct {
path string
artifact *pipeline.ArtifactReference
}
func (source *fileReferenceSource) UnmarshalYAML(node *yaml.Node) error {
if source == nil {
return fmt.Errorf("reference source must not be nil")
}
switch node.Kind {
case yaml.ScalarNode:
if node.Tag != "!!str" {
return fmt.Errorf("external reference path must be a string")
}
path := strings.TrimSpace(node.Value)
if path == "" {
return fmt.Errorf("external reference path must not be empty")
}
source.path = path
source.artifact = nil
return nil
case yaml.MappingNode:
if len(node.Content) != 2 || node.Content[0].Value != "artifact" {
return fmt.Errorf("reference source mapping must contain only artifact")
}
artifactNode := node.Content[1]
if artifactNode.Kind != yaml.MappingNode {
return fmt.Errorf("artifact reference must be an object")
}
var step, lane string
seen := map[string]bool{}
for i := 0; i < len(artifactNode.Content); i += 2 {
key := artifactNode.Content[i].Value
value := artifactNode.Content[i+1]
if seen[key] {
return fmt.Errorf("artifact reference field %q is duplicated", key)
}
seen[key] = true
if value.Tag != "!!str" {
return fmt.Errorf("artifact reference field %q must be a string", key)
}
switch key {
case "step":
step = strings.TrimSpace(value.Value)
case "lane":
lane = strings.TrimSpace(value.Value)
default:
return fmt.Errorf("field %s not found in artifact reference", key)
}
}
if step == "" || lane == "" {
return fmt.Errorf("artifact reference step and lane must not be empty")
}
source.path = ""
source.artifact = &pipeline.ArtifactReference{Step: step, Lane: lane}
return nil
default:
return fmt.Errorf("reference source must be a string or object")
}
}
func (source fileReferenceSource) toPipelineSource() pipeline.ReferenceSource {
if source.artifact != nil {
artifact := *source.artifact
return pipeline.ReferenceSource{Artifact: &artifact}
}
return pipeline.ExternalReference(source.path)
}
func fileReferenceSourcesToPipeline(values map[string]fileReferenceSource) map[string]pipeline.ReferenceSource {
if len(values) == 0 {
return nil
}
out := make(map[string]pipeline.ReferenceSource, len(values))
for key, value := range values {
out[strings.TrimSpace(key)] = value.toPipelineSource()
}
return out
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
var module string
if err := node.Decode(&module); err != nil {
return fmt.Errorf("module binding must be a string or object")
}
b.Module = strings.TrimSpace(module)
return nil
case yaml.MappingNode:
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valueNode := node.Content[i+1]
switch keyNode.Value {
case "module":
var module string
if err := valueNode.Decode(&module); err != nil {
return err
}
b.Module = strings.TrimSpace(module)
case "llm_profile":
var llmProfile string
if err := valueNode.Decode(&llmProfile); err != nil {
return err
}
b.LLMProfile = strings.TrimSpace(llmProfile)
if b.LLMProfile == "" {
return fmt.Errorf("llm_profile must not be empty when set")
}
case "structured_output_repair_attempts":
attempts, err := parseStructuredOutputRepairAttempts(valueNode, "module binding")
if err != nil {
return err
}
b.StructuredOutputRepairAttempts = attempts
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
return err
}
b.Retries = retries
case "options":
var options map[string]any
if err := valueNode.Decode(&options); err != nil {
return err
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]fileReferenceSource
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
case "validators":
b.Validators.Set = true
var validators []fileModuleBinding
if err := valueNode.Decode(&validators); err != nil {
return err
}
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
for i, validator := range validators {
b.Validators.Validators[i] = validator.toPipelineBinding()
}
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
}
return nil
default:
return fmt.Errorf("module binding must be a string or object")
}
}
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
return pipeline.ModuleBinding{
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References),
Validators: cloneValidatorOverride(b.Validators),
}
}
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("%s must be an object", context)
}
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value != "structured_output_repair_attempts" {
continue
}
if _, err := parseStructuredOutputRepairAttempts(node.Content[i+1], context); err != nil {
return err
}
}
return nil
}
func parseStructuredOutputRepairAttempts(node *yaml.Node, context string) (*int, error) {
if node.Tag == "!!null" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must not be null", context)
}
if node.Kind != yaml.ScalarNode || node.Tag != "!!int" {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer", context)
}
var attempts int
if err := node.Decode(&attempts); err != nil {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be an integer: %w", context, err)
}
if attempts < 0 || attempts > 3 {
return nil, fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return &attempts, nil
}
func cloneStructuredOutputRepairAttempts(attempts *int) *int {
if attempts == nil {
return nil
}
value := *attempts
return &value
}
func LoadFileConfig(path string) (FileConfig, error) {
data, err := os.ReadFile(path)
if err != nil {
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
}
cfg, err := ParseFileConfigYAML(data)
if err != nil {
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
}
return cfg, nil
}
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
var header struct {
Version int `yaml:"version"`
}
if err := yaml.Unmarshal(data, &header); err != nil {
return FileConfig{}, fmt.Errorf("decode yaml version header: %w", err)
}
if header.Version == 0 {
return FileConfig{}, fmt.Errorf("config version is required")
}
if header.Version == 2 {
return FileConfig{}, fmt.Errorf("config version 2 is no longer supported; migrate the file using the version 2-to-3 migration in docs/config.md")
}
if header.Version == 3 {
return FileConfig{}, fmt.Errorf("config version 3 is no longer supported; change \"version: 3\" to \"version: 4\" and rename \"scriptorium:\" to \"promptkit:\"")
}
if header.Version != SupportedFileConfigVersion {
return FileConfig{}, fmt.Errorf("unsupported config version %d (supported version is %d)", header.Version, SupportedFileConfigVersion)
}
var fileCfg FileConfig
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&fileCfg); err != nil {
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
}
var trailing any
if err := decoder.Decode(&trailing); err == nil {
return FileConfig{}, fmt.Errorf("config must contain exactly one YAML document")
} else if err != io.EOF {
return FileConfig{}, fmt.Errorf("decode trailing yaml document: %w", err)
}
return fileCfg, nil
}
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
}
func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
return c.applyFileConfigWithLookup(fileCfg, lookup)
}
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
_ = lookup
if c == nil {
return fmt.Errorf("config must not be nil")
}
if fileCfg.Version != SupportedFileConfigVersion {
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
if c.Pipelines == nil {
c.Pipelines = map[string]pipeline.PipelineProfile{}
}
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
if err != nil {
return err
}
for _, pipelineID := range pipelineIDs {
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
hasArtifacts := filePipeline.artifactsSet || filePipeline.Artifacts != nil
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
if hasArtifacts && hasSteps {
return fmt.Errorf("pipeline %q must not declare both artifacts and steps", pipelineID)
}
if hasSteps && len(filePipeline.Steps) == 0 {
return fmt.Errorf("pipeline %q must declare at least one ordered step", pipelineID)
}
if hasSteps {
seenSteps := make(map[string]struct{}, len(filePipeline.Steps))
seenLanes := make(map[string]struct{})
for index, step := range filePipeline.Steps {
stepID := strings.TrimSpace(step.ID)
if stepID == "" {
return fmt.Errorf("pipeline %q step[%d] id must not be empty", pipelineID, index)
}
if _, ok := seenSteps[stepID]; ok {
return fmt.Errorf("pipeline %q step id %q is duplicated after trimming", pipelineID, stepID)
}
seenSteps[stepID] = struct{}{}
laneIDs, rawLaneIDs, err := normalizedMapKeys(step.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
if err != nil {
return err
}
for _, laneID := range laneIDs {
if _, ok := seenLanes[laneID]; ok {
return fmt.Errorf("pipeline %q artifact lane id %q is duplicated across steps", pipelineID, laneID)
}
seenLanes[laneID] = struct{}{}
fileLane := step.Artifacts[rawLaneIDs[laneID]]
if err := validateFileLaneReferences(pipelineID, stepID, laneID, fileLane); err != nil {
return err
}
}
if err := validateFileReferenceSources(step.References, fmt.Sprintf("pipeline %q step %q reference slot", pipelineID, stepID)); err != nil {
return err
}
}
}
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err
}
if err := validateFileReferenceSources(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
}
if fileCfg.PromptKit != nil {
if fileCfg.PromptKit.ProfileDir != nil {
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileDir)
if value == "" {
return fmt.Errorf("promptkit.profile_dir must not be empty when set")
}
c.PromptKit.ProfileDir = value
}
if fileCfg.PromptKit.ProfileFile != nil {
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileFile)
if value == "" {
return fmt.Errorf("promptkit.profile_file must not be empty when set")
}
c.PromptKit.ProfileFile = value
}
if fileCfg.PromptKit.LocalBackend != nil {
if fileCfg.PromptKit.LocalBackend.Endpoint == nil {
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
}
endpoint := strings.TrimSpace(*fileCfg.PromptKit.LocalBackend.Endpoint)
if endpoint == "" {
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
}
localBackend := PromptKitLocalBackendConfig{Endpoint: endpoint}
if fileCfg.PromptKit.LocalBackend.ConcurrencyLimit != nil {
localBackend.ConcurrencyLimit = *fileCfg.PromptKit.LocalBackend.ConcurrencyLimit
}
c.PromptKit.LocalBackend = &localBackend
}
}
for _, pipelineID := range pipelineIDs {
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
llmProfile := ""
if filePipeline.llmProfileSet || filePipeline.LLMProfile != nil {
if filePipeline.LLMProfile == nil || strings.TrimSpace(*filePipeline.LLMProfile) == "" {
return fmt.Errorf("pipeline %q llm_profile must not be empty when set", pipelineID)
}
llmProfile = strings.TrimSpace(*filePipeline.LLMProfile)
}
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
if err != nil {
return err
}
profile := pipeline.PipelineProfile{
ID: pipelineID,
LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
}
if filePipeline.Output != nil {
profile.Output = filePipeline.Output.toPipelineBinding()
}
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: extract,
References: fileReferenceSourcesToPipeline(fileLane.References),
}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
}
if fileLane.Normalize != nil {
lane.Normalize = fileLane.Normalize.toPipelineBinding()
}
if len(fileLane.Validators) > 0 {
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
for i, validator := range fileLane.Validators {
lane.Validators[i] = validator.toPipelineBinding()
}
}
profile.Artifacts[laneID] = lane
}
if hasSteps {
profile.Artifacts = nil
profile.Steps = make([]pipeline.PipelineStepProfile, len(filePipeline.Steps))
for i, fileStep := range filePipeline.Steps {
stepID := strings.TrimSpace(fileStep.ID)
step := pipeline.PipelineStepProfile{
ID: stepID,
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(fileStep.Artifacts)),
References: fileReferenceSourcesToPipeline(fileStep.References),
}
stepLaneIDs, stepRawLaneIDs, err := normalizedMapKeys(fileStep.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
if err != nil {
return err
}
for _, laneID := range stepLaneIDs {
fileLane := fileStep.Artifacts[stepRawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{Extract: extract, References: fileReferenceSourcesToPipeline(fileLane.References)}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
}
if fileLane.Normalize != nil {
lane.Normalize = fileLane.Normalize.toPipelineBinding()
}
if len(fileLane.Validators) > 0 {
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
for index, validator := range fileLane.Validators {
lane.Validators[index] = validator.toPipelineBinding()
}
}
step.Artifacts[laneID] = lane
}
profile.Steps[i] = step
}
}
c.Pipelines[pipelineID] = profile
}
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
}
if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil {
workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers)
if err != nil {
return err
}
c.Concurrency.StageWorkers = workers
c.Concurrency.extractWorkersConfigured = configured
}
c.Concurrency.recomputeStageWorkerDefaults()
if fileCfg.Output != nil && fileCfg.Output.Directory != nil {
c.Output.Directory = strings.TrimSpace(*fileCfg.Output.Directory)
if c.Output.Directory == "" {
return fmt.Errorf("output.directory must not be empty")
}
if strings.ContainsRune(c.Output.Directory, '\x00') {
return fmt.Errorf("output.directory must not contain NUL")
}
}
if fileCfg.Cache != nil {
if fileCfg.Cache.ChunkPlans != nil {
if fileCfg.Cache.ChunkPlans.Mode != nil {
mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Cache.ChunkPlans.Mode)
if err != nil {
return fmt.Errorf("cache.chunk_plans.mode: %w", err)
}
c.Cache.ChunkPlans.Mode = mode
}
if fileCfg.Cache.ChunkPlans.Directory != nil {
c.Cache.ChunkPlans.Directory = cleanOptionalPath(*fileCfg.Cache.ChunkPlans.Directory)
if strings.ContainsRune(c.Cache.ChunkPlans.Directory, '\x00') {
return fmt.Errorf("cache.chunk_plans.directory must not contain NUL")
}
}
}
if fileCfg.Cache.Checkpoints != nil {
if fileCfg.Cache.Checkpoints.Enabled != nil {
c.Cache.Checkpoints.Enabled = *fileCfg.Cache.Checkpoints.Enabled
}
if fileCfg.Cache.Checkpoints.Directory != nil {
c.Cache.Checkpoints.Directory = cleanOptionalPath(*fileCfg.Cache.Checkpoints.Directory)
if strings.ContainsRune(c.Cache.Checkpoints.Directory, '\x00') {
return fmt.Errorf("cache.checkpoints.directory must not contain NUL")
}
}
}
}
if fileCfg.Debug != nil && fileCfg.Debug.Directory != nil {
c.Debug.Directory = strings.TrimSpace(*fileCfg.Debug.Directory)
if c.Debug.Directory == "" {
return fmt.Errorf("debug.directory must not be empty")
}
if strings.ContainsRune(c.Debug.Directory, '\x00') {
return fmt.Errorf("debug.directory must not contain NUL")
}
}
return nil
}
func cleanOptionalPath(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
return filepath.Clean(value)
}
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
workers := make(map[string]int, len(values))
configured := false
for rawKey, value := range values {
key := strings.TrimSpace(rawKey)
if key == "" {
return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey)
}
if _, exists := workers[key]; exists {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key)
}
workers[key] = value
configured = true
}
return workers, configured, nil
}
func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawID := range values {
id := strings.TrimSpace(rawID)
if id == "" {
return nil, nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := rawByNormalized[id]; ok {
return nil, nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, id)
}
rawByNormalized[id] = rawID
keys = append(keys, id)
}
sort.Strings(keys)
return keys, rawByNormalized, nil
}
func validateFileReferenceSources(values map[string]fileReferenceSource, context string) error {
seen := make(map[string]struct{}, len(values))
for rawSlot, source := range values {
slot := strings.TrimSpace(rawSlot)
if slot == "" {
return fmt.Errorf("%s must not be empty", context)
}
if _, ok := seen[slot]; ok {
return fmt.Errorf("%s %q is duplicated after trimming", context, slot)
}
seen[slot] = struct{}{}
if source.artifact != nil {
if strings.TrimSpace(source.artifact.Step) == "" || strings.TrimSpace(source.artifact.Lane) == "" {
return fmt.Errorf("%s %q artifact selector step and lane must not be empty", context, slot)
}
if strings.TrimSpace(source.path) != "" {
return fmt.Errorf("%s %q must contain either an external path or artifact selector", context, slot)
}
continue
}
if strings.TrimSpace(source.path) == "" {
return fmt.Errorf("%s %q source must not be empty", context, slot)
}
}
return nil
}
func validateFileLaneReferences(pipelineID, stepID, laneID string, lane FileArtifactLaneProfile) error {
prefix := fmt.Sprintf("pipeline %q step %q lane %q", pipelineID, stepID, laneID)
references := []struct {
label string
values map[string]fileReferenceSource
}{
{label: "reference slot", values: lane.References},
{label: "extract reference slot", values: lane.Extract.References},
}
if lane.Merge != nil {
references = append(references, struct {
label string
values map[string]fileReferenceSource
}{label: "merge reference slot", values: lane.Merge.References})
}
if lane.Normalize != nil {
references = append(references, struct {
label string
values map[string]fileReferenceSource
}{label: "normalize reference slot", values: lane.Normalize.References})
}
for _, item := range references {
if err := validateFileReferenceSources(item.values, prefix+" "+item.label); err != nil {
return err
}
}
for index, validator := range lane.Validators {
if err := validateFileReferenceSources(validator.References, fmt.Sprintf("%s validator[%d] reference slot", prefix, index)); err != nil {
return err
}
}
return nil
}
func mergeReferenceSources(base map[string]pipeline.ReferenceSource, override map[string]pipeline.ReferenceSource) map[string]pipeline.ReferenceSource {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]pipeline.ReferenceSource, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func normalizeOptions(options map[string]any) map[string]any {
if len(options) == 0 {
return nil
}
return cloneOptions(options)
}

View File

@@ -1,928 +0,0 @@
package config
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestDefaultReturnsDocumentedValuesAndIndependentMaps(t *testing.T) {
first := Default()
if first.Concurrency.TotalLLM != 16 || first.Concurrency.StageWorkers["extract"] != 16 {
t.Fatalf("concurrency defaults = %#v", first.Concurrency)
}
if first.Output.Directory != "./notarius-output" || first.Debug.Directory != "./notarius-debug" {
t.Fatalf("output/debug defaults = %#v, %#v", first.Output, first.Debug)
}
if first.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto || first.Cache.ChunkPlans.Directory != "" || first.Cache.Checkpoints.Enabled || first.Cache.Checkpoints.Directory != "" {
t.Fatalf("cache defaults = %#v", first.Cache)
}
if len(first.Pipelines) != 0 {
t.Fatalf("pipeline defaults = %#v", first.Pipelines)
}
first.Concurrency.StageWorkers["extract"] = 99
first.Concurrency.StageWorkers["other"] = 100
first.Pipelines["changed"] = pipeline.PipelineProfile{}
second := Default()
if second.Concurrency.StageWorkers["extract"] != 16 || len(second.Concurrency.StageWorkers) != 1 || len(second.Pipelines) != 0 {
t.Fatalf("Default() returned state shared with an earlier result: %#v", second)
}
}
func TestFileConfigMinimalVersion4AppliesOverDefaults(t *testing.T) {
file := parseFileConfig(t, "version: 4\n")
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if cfg.Output.Directory != "./notarius-output" || cfg.Debug.Directory != "./notarius-debug" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto {
t.Fatalf("minimal file changed unrelated defaults: %#v", cfg)
}
if cfg.Concurrency.TotalLLM != 16 || cfg.Concurrency.StageWorkers["extract"] != 16 || len(cfg.Pipelines) != 0 {
t.Fatalf("minimal file did not retain defaults: %#v", cfg)
}
}
func TestParseFileConfigYAMLRejectsAdditionalDocuments(t *testing.T) {
tests := []struct {
name string
source string
wantErr bool
}{
{name: "trailing whitespace", source: "version: 4\n\n \t", wantErr: false},
{name: "trailing comment", source: "version: 4\n# trailing comment\n", wantErr: false},
{name: "second valid document", source: "version: 4\n---\nversion: 4\n", wantErr: true},
{name: "second empty document", source: "version: 4\n---\n", wantErr: true},
{name: "second malformed document", source: "version: 4\n---\nversion: [\n", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(tt.source))
if (err != nil) != tt.wantErr {
t.Fatalf("ParseFileConfigYAML() error = %v, want error=%t", err, tt.wantErr)
}
})
}
}
func TestFilePipelineLLMProfileIsPresenceAwareAndDetached(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
%s
input: input
artifacts:
lane:
extract: extract
`
t.Run("omitted", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
if file.Pipelines["main"].LLMProfile != nil || file.Pipelines["main"].llmProfileSet {
t.Fatalf("parsed pipeline profile = %#v, want omitted llm profile", file.Pipelines["main"])
}
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].LLMProfile; got != "" {
t.Fatalf("pipeline llm profile = %q, want empty", got)
}
})
t.Run("trimmed and detached", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: ' configured-profile '"))
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("pipeline llm profile = %q, want trimmed value", got)
}
*file.Pipelines["main"].LLMProfile = "changed-profile"
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("effective config aliases parsed file: %q", got)
}
if got := cloneConfig(cfg).Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("cloned pipeline llm profile = %q", got)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatal(err)
}
var roundTripped Config
if err := json.Unmarshal(data, &roundTripped); err != nil {
t.Fatal(err)
}
if got := roundTripped.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("round-tripped pipeline llm profile = %q", got)
}
})
for _, value := range []string{"''", "' '", "null"} {
t.Run("explicit empty "+value, func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: "+value))
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), `pipeline "main" llm_profile must not be empty`) {
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
}
})
}
}
func TestStructuredOutputRepairAttemptsFileConfigurationPreservesPresenceAndOwnership(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
%s
input: input
artifacts:
lane:
extract:
module: extract
structured_output_repair_attempts: 2
validators:
- module: validator
structured_output_repair_attempts: 3
`
t.Run("omitted pipeline value remains absent", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
if got := file.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("file pipeline repair attempts = %v, want nil", *got)
}
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != nil {
t.Fatalf("pipeline repair attempts = %v, want nil", *got)
}
if got := cfg.Pipelines["main"].Input.StructuredOutputRepairAttempts; got != nil {
t.Fatalf("scalar input binding repair attempts = %v, want nil", *got)
}
})
t.Run("zero is explicit and survives configuration boundaries", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "structured_output_repair_attempts: 0"))
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
profile := cfg.Pipelines["main"]
if profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("pipeline repair attempts = %v, want explicit zero", profile.StructuredOutputRepairAttempts)
}
lane := profile.Artifacts["lane"]
if lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("extract repair attempts = %v, want 2", lane.Extract.StructuredOutputRepairAttempts)
}
if lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("validator repair attempts = %v, want 3", lane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
*file.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("applied config aliased file configuration: got %d, want 0", got)
}
fileProfile := file.Pipelines["main"]
fileLane := fileProfile.Artifacts["lane"]
*fileLane.Extract.StructuredOutputRepairAttempts = 1
*fileLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts = 1
fileProfile.Artifacts["lane"] = fileLane
file.Pipelines["main"] = fileProfile
configuredLane := cfg.Pipelines["main"].Artifacts["lane"]
if got := *configuredLane.Extract.StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("configured extract aliased file configuration: got %d, want 2", got)
}
if got := *configuredLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("configured validator aliased file configuration: got %d, want 3", got)
}
cloned := cloneConfig(cfg)
*cloned.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("cloned config aliased source configuration: got %d, want 0", got)
}
redacted := cfg.Redacted()
*redacted.Pipelines["main"].StructuredOutputRepairAttempts = 1
if got := *cfg.Pipelines["main"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("redacted config aliased source configuration: got %d, want 0", got)
}
summary := cfg.RedactedSummaryPayload().(Config)
if summary.Pipelines["main"].StructuredOutputRepairAttempts == nil || *summary.Pipelines["main"].StructuredOutputRepairAttempts != 0 {
t.Fatalf("redacted summary pipeline repair attempts = %v, want explicit zero", summary.Pipelines["main"].StructuredOutputRepairAttempts)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatal(err)
}
var roundTripped Config
if err := json.Unmarshal(data, &roundTripped); err != nil {
t.Fatal(err)
}
roundTrippedProfile := roundTripped.Pipelines["main"]
if roundTrippedProfile.StructuredOutputRepairAttempts == nil || *roundTrippedProfile.StructuredOutputRepairAttempts != 0 {
t.Fatalf("round-tripped pipeline repair attempts = %v, want explicit zero", roundTrippedProfile.StructuredOutputRepairAttempts)
}
roundTrippedLane := roundTrippedProfile.Artifacts["lane"]
if roundTrippedLane.Extract.StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.StructuredOutputRepairAttempts != 2 {
t.Fatalf("round-tripped extract repair attempts = %v, want 2", roundTrippedLane.Extract.StructuredOutputRepairAttempts)
}
if roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts == nil || *roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts != 3 {
t.Fatalf("round-tripped validator repair attempts = %v, want 3", roundTrippedLane.Extract.Validators.Validators[0].StructuredOutputRepairAttempts)
}
})
}
func TestStructuredOutputRepairAttemptsFileConfigurationRejectsInvalidValues(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
input: input
artifacts:
lane:
extract: extract
%s
`
const bindingYAML = `version: 4
pipelines:
main:
input:
module: input
%s
artifacts:
lane:
extract: extract
`
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null pipeline value", source: "structured_output_repair_attempts: null", want: "pipeline profile structured_output_repair_attempts must not be null"},
{name: "fractional pipeline value", source: "structured_output_repair_attempts: 1.5", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "quoted pipeline value", source: "structured_output_repair_attempts: '1'", want: "pipeline profile structured_output_repair_attempts must be an integer"},
{name: "out of range pipeline value", source: "structured_output_repair_attempts: 4", want: "pipeline profile structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(pipelineYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
for _, tt := range []struct {
name string
source string
want string
}{
{name: "null binding value", source: "structured_output_repair_attempts: null", want: "module binding structured_output_repair_attempts must not be null"},
{name: "noninteger binding value", source: "structured_output_repair_attempts: true", want: "module binding structured_output_repair_attempts must be an integer"},
{name: "out of range binding value", source: "structured_output_repair_attempts: -1", want: "module binding structured_output_repair_attempts must be between zero and three"},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(bindingYAML, tt.source)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ParseFileConfigYAML() error = %v, want %q", err, tt.want)
}
})
}
}
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
const configYAML = `version: 4
pipelines:
main:
input:
module: input
llm_profile: %s
artifacts:
lane:
extract: extract
`
for _, value := range []string{"''", "' '", "null"} {
t.Run(value, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(configYAML, value)))
if err == nil || !strings.Contains(err.Error(), "llm_profile must not be empty when set") {
t.Fatalf("ParseFileConfigYAML() error = %v, want explicit-empty binding profile rejection", err)
}
})
}
}
func TestFilePromptKitProfileSourcesSurviveConfigBoundaries(t *testing.T) {
tests := []struct {
name string
yaml string
want PromptKitConfig
}{
{
name: "profile directory",
yaml: "version: 4\npromptkit:\n profile_dir: ' ./profiles '\n",
want: PromptKitConfig{ProfileDir: "./profiles"},
},
{
name: "profile file",
yaml: "version: 4\npromptkit:\n profile_file: ' ./profiles.yml '\n",
want: PromptKitConfig{ProfileFile: "./profiles.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := applyFileConfig(t, tt.yaml)
if cfg.PromptKit != tt.want {
t.Fatalf("PromptKit config = %#v, want %#v", cfg.PromptKit, tt.want)
}
if got := cloneConfig(cfg).PromptKit; got != tt.want {
t.Fatalf("cloned PromptKit config = %#v, want %#v", got, tt.want)
}
if got := cfg.Redacted().PromptKit; got != tt.want {
t.Fatalf("redacted PromptKit config = %#v, want %#v", got, tt.want)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if _, ok := payload["promptkit"]; !ok {
t.Fatalf("runtime JSON keys = %v, want promptkit", payload)
}
if _, ok := payload["scriptorium"]; ok {
t.Fatalf("runtime JSON keys = %v, must not contain removed section", payload)
}
})
}
}
func TestFilePromptKitLocalBackendSurvivesConfigBoundaries(t *testing.T) {
tests := []struct {
name string
concurrencyYAML string
wantConcurrency int
}{
{name: "omitted concurrency defaults to zero"},
{name: "positive concurrency is preserved", concurrencyYAML: " concurrency_limit: 2\n", wantConcurrency: 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, "version: 4\npromptkit:\n local_backend:\n endpoint: ' http://localhost:8000/v1 '\n"+tt.concurrencyYAML)
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
want := PromptKitLocalBackendConfig{
Endpoint: "http://localhost:8000/v1",
ConcurrencyLimit: tt.wantConcurrency,
}
if cfg.PromptKit.LocalBackend == nil || *cfg.PromptKit.LocalBackend != want {
t.Fatalf("local backend config = %#v, want %#v", cfg.PromptKit.LocalBackend, want)
}
*file.PromptKit.LocalBackend.Endpoint = "http://changed.example/v1"
if file.PromptKit.LocalBackend.ConcurrencyLimit != nil {
*file.PromptKit.LocalBackend.ConcurrencyLimit = 99
}
if *cfg.PromptKit.LocalBackend != want {
t.Fatalf("effective config aliases parsed file model: %#v", cfg.PromptKit.LocalBackend)
}
cloned := cloneConfig(cfg)
if cloned.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *cloned.PromptKit.LocalBackend != want {
t.Fatalf("cloned local backend = %#v, want detached %#v", cloned.PromptKit.LocalBackend, want)
}
cloned.PromptKit.LocalBackend.Endpoint = "http://clone.example/v1"
if *cfg.PromptKit.LocalBackend != want {
t.Fatalf("mutating clone changed source config: %#v", cfg.PromptKit.LocalBackend)
}
redacted := cfg.Redacted()
if redacted.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *redacted.PromptKit.LocalBackend != want {
t.Fatalf("redacted local backend = %#v, want detached %#v", redacted.PromptKit.LocalBackend, want)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var payload struct {
PromptKit map[string]json.RawMessage `json:"promptkit"`
}
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
localJSON, ok := payload.PromptKit["local_backend"]
if !ok {
t.Fatalf("runtime PromptKit JSON keys = %v, want local_backend", payload.PromptKit)
}
var localPayload map[string]json.RawMessage
if err := json.Unmarshal(localJSON, &localPayload); err != nil {
t.Fatalf("unmarshal local_backend JSON: %v", err)
}
if _, ok := localPayload["endpoint"]; !ok {
t.Fatalf("runtime local_backend JSON keys = %v, want endpoint", localPayload)
}
if _, ok := localPayload["concurrency_limit"]; !ok {
t.Fatalf("runtime local_backend JSON keys = %v, want concurrency_limit", localPayload)
}
})
}
}
func TestFilePromptKitLocalBackendRequiresEndpoint(t *testing.T) {
for _, tt := range []struct {
name string
yaml string
}{
{name: "missing", yaml: "version: 4\npromptkit:\n local_backend: {}\n"},
{name: "empty", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ''\n"},
{name: "blank", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ' '\n"},
} {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, tt.yaml)
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), "promptkit.local_backend.endpoint") {
t.Fatalf("ApplyFileConfig() error = %v, want endpoint field context", err)
}
})
}
}
func TestFilePromptKitExplicitEmptyProfileSourcesAreRejected(t *testing.T) {
for _, field := range []string{"profile_dir", "profile_file"} {
t.Run(field, func(t *testing.T) {
file := parseFileConfig(t, "version: 4\npromptkit:\n "+field+": ''\n")
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), "promptkit."+field+" must not be empty") {
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
}
})
}
}
func TestFilePromptKitProfileSourcesRemainMutuallyExclusive(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
promptkit:
profile_dir: ./profiles
profile_file: ./profiles.yml
`)
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "promptkit profile_dir and profile_file are mutually exclusive") {
t.Fatalf("Validate() error = %v, want mutually exclusive profile sources", err)
}
}
func TestFileConfigMissingVersionIsReportedBeforeFieldDecoding(t *testing.T) {
_, err := ParseFileConfigYAML([]byte("workspace:\n directory: /tmp/old\n"))
if err == nil || !strings.Contains(err.Error(), "config version is required") {
t.Fatalf("missing version error = %v", err)
}
}
func TestFileConfigVersion3ReportsPromptKitMigration(t *testing.T) {
_, err := ParseFileConfigYAML([]byte("version: 3\nscriptorium:\n profile_dir: ./profiles\n"))
if err == nil ||
!strings.Contains(err.Error(), `change "version: 3" to "version: 4"`) ||
!strings.Contains(err.Error(), `rename "scriptorium:" to "promptkit:"`) {
t.Fatalf("version 3 error = %v, want actionable version and section migration", err)
}
}
func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{
name: "removed diagnostics",
yaml: "version: 4\ndiagnostics: {}\n",
want: "field diagnostics not found",
},
{
name: "removed llm profiles",
yaml: "version: 4\nllm_profiles: {}\n",
want: "field llm_profiles not found",
},
{
name: "removed scriptorium section",
yaml: "version: 4\nscriptorium: {}\n",
want: "field scriptorium not found",
},
{
name: "version 2 migration",
yaml: "version: 2\nworkspace:\n directory: /tmp/old\n",
want: "version 2-to-3 migration",
},
{
name: "pipeline field",
yaml: "version: 4\npipelines:\n main:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "lane field",
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "module binding field",
yaml: "version: 4\npipelines:\n main:\n input:\n module: seriatim\n unknown: true\n",
want: "field unknown not found in module binding",
},
{
name: "checkpoint field",
yaml: "version: 4\ncache:\n checkpoints:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "checkpoint enabled type",
yaml: "version: 4\ncache:\n checkpoints:\n enabled: definitely\n",
want: "cannot unmarshal",
},
{
name: "local backend field",
yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: http://localhost:8000/v1\n unknown: true\n",
want: "field unknown not found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(tt.yaml))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %v, want context %q", err, tt.want)
}
})
}
}
func TestFileConfigModuleBindingsPreserveFormsAndValidatorPresence(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
chunk:
module: generic
llm_profile: chunk-profile
retries: 2
options:
max_units: 25
references:
glossary: ./glossary.md
validators: []
artifacts:
spells:
extract:
module: dnd/spells
options:
nested:
enabled: true
merge: appendorder
normalize: noop
`)
profile := cfg.Pipelines["main"]
if profile.Input.Module != "seriatim" || profile.Input.Validators.Set {
t.Fatalf("shorthand binding = %#v", profile.Input)
}
if profile.Chunk.Module != "generic" || profile.Chunk.LLMProfile != "chunk-profile" || profile.Chunk.Retries != 2 ||
!reflect.DeepEqual(profile.Chunk.Options, map[string]any{"max_units": 25}) ||
!reflect.DeepEqual(profile.Chunk.References, pipeline.ExternalReferenceMap(map[string]string{"glossary": "./glossary.md"})) {
t.Fatalf("object binding = %#v", profile.Chunk)
}
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
t.Fatalf("explicit empty validators = %#v", profile.Chunk.Validators)
}
if profile.Artifacts["spells"].Extract.Module != "dnd/spells" ||
!reflect.DeepEqual(profile.Artifacts["spells"].Extract.Options, map[string]any{
"nested": map[string]any{"enabled": true},
}) {
t.Fatalf("extract binding = %#v", profile.Artifacts["spells"].Extract)
}
if profile.Artifacts["spells"].Merge.Module != "appendorder" || profile.Artifacts["spells"].Normalize.Module != "noop" {
t.Fatalf("stage shorthand bindings = %#v", profile.Artifacts["spells"])
}
}
func TestFileConfigReferencePrecedenceIsRetained(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
references:
pipeline-only: ./pipeline.txt
shared: ./pipeline-shared.txt
chunk:
module: generic
references:
chunk-only: ./chunk.txt
artifacts:
spells:
references:
lane-only: ./lane.txt
shared: ./lane-shared.txt
overridden: ./lane.txt
extract:
module: dnd/spells
references:
extract-only: ./extract.txt
overridden: ./extract-overridden.txt
merge:
module: appendorder
references:
merge-only: ./merge.txt
normalize:
module: noop
references:
normalize-only: ./normalize.txt
`)
profile := cfg.Pipelines["main"]
if !reflect.DeepEqual(profile.References, pipeline.ExternalReferenceMap(map[string]string{
"pipeline-only": "./pipeline.txt",
"shared": "./pipeline-shared.txt",
})) {
t.Fatalf("pipeline references = %#v", profile.References)
}
if !reflect.DeepEqual(profile.Chunk.References, pipeline.ExternalReferenceMap(map[string]string{"chunk-only": "./chunk.txt"})) {
t.Fatalf("chunk references = %#v", profile.Chunk.References)
}
lane := profile.Artifacts["spells"]
if !reflect.DeepEqual(lane.References, pipeline.ExternalReferenceMap(map[string]string{
"lane-only": "./lane.txt",
"shared": "./lane-shared.txt",
"overridden": "./lane.txt",
})) {
t.Fatalf("lane compatibility references = %#v", lane.References)
}
if !reflect.DeepEqual(lane.Extract.References, pipeline.ExternalReferenceMap(map[string]string{
"lane-only": "./lane.txt",
"shared": "./lane-shared.txt",
"overridden": "./extract-overridden.txt",
"extract-only": "./extract.txt",
})) {
t.Fatalf("extract references = %#v", lane.Extract.References)
}
if !reflect.DeepEqual(lane.Merge.References, pipeline.ExternalReferenceMap(map[string]string{"merge-only": "./merge.txt"})) ||
!reflect.DeepEqual(lane.Normalize.References, pipeline.ExternalReferenceMap(map[string]string{"normalize-only": "./normalize.txt"})) {
t.Fatalf("merge/normalize references = %#v, %#v", lane.Merge.References, lane.Normalize.References)
}
}
func TestFileConfigStageLocalValidatorsPreserveOrderAndFields(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
chunk:
module: generic
validators:
- generic/always_accept
- module: generic/valid_json
llm_profile: validator-profile
options:
schema: compact
artifacts:
spells:
extract:
module: dnd/spells
validators:
- module: extract/dnd/spells/shape
options:
strict: true
merge:
module: appendorder
validators:
- generic/always_accept
normalize:
module: noop
validators:
- module: generic/valid_json
options:
mode: normalized
`)
profile := cfg.Pipelines["main"]
chunkValidators := profile.Chunk.Validators.Validators
if !profile.Chunk.Validators.Set || len(chunkValidators) != 2 || chunkValidators[0].Module != "generic/always_accept" ||
chunkValidators[1].Module != "generic/valid_json" || chunkValidators[1].LLMProfile != "validator-profile" ||
!reflect.DeepEqual(chunkValidators[1].Options, map[string]any{"schema": "compact"}) {
t.Fatalf("chunk validators = %#v", profile.Chunk.Validators)
}
lane := profile.Artifacts["spells"]
if len(lane.Extract.Validators.Validators) != 1 || lane.Extract.Validators.Validators[0].Module != "extract/dnd/spells/shape" ||
!reflect.DeepEqual(lane.Extract.Validators.Validators[0].Options, map[string]any{"strict": true}) {
t.Fatalf("extract validators = %#v", lane.Extract.Validators)
}
if len(lane.Merge.Validators.Validators) != 1 || lane.Merge.Validators.Validators[0].Module != "generic/always_accept" {
t.Fatalf("merge validators = %#v", lane.Merge.Validators)
}
if len(lane.Normalize.Validators.Validators) != 1 || lane.Normalize.Validators.Validators[0].Module != "generic/valid_json" ||
!reflect.DeepEqual(lane.Normalize.Validators.Validators[0].Options, map[string]any{"mode": "normalized"}) {
t.Fatalf("normalize validators = %#v", lane.Normalize.Validators)
}
}
func TestFileConfigStateSectionsApplyIndependently(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
promptkit:
profile_dir: ./profiles
concurrency:
total_llm: 7
output:
directory: ./output
cache:
chunk_plans:
directory: ./plans
mode: bypass
checkpoints:
enabled: true
directory: ./checkpoints
debug:
directory: ./debug
`)
if cfg.PromptKit.ProfileDir != "./profiles" || cfg.PromptKit.ProfileFile != "" {
t.Fatalf("promptkit = %#v", cfg.PromptKit)
}
if cfg.Concurrency.TotalLLM != 7 || cfg.Concurrency.StageWorkers["extract"] != 7 {
t.Fatalf("concurrency = %#v", cfg.Concurrency)
}
if cfg.Output.Directory != "./output" || cfg.Cache.ChunkPlans.Directory != "plans" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass ||
!cfg.Cache.Checkpoints.Enabled || cfg.Cache.Checkpoints.Directory != "checkpoints" || cfg.Debug.Directory != "./debug" {
t.Fatalf("state sections = %#v, %#v, %#v, %#v", cfg.Output, cfg.Cache, cfg.Debug, cfg.PromptKit)
}
if cfg.Output.Directory == cfg.Cache.ChunkPlans.Directory || cfg.Cache.ChunkPlans.Directory == cfg.Cache.Checkpoints.Directory || cfg.Cache.Checkpoints.Directory == cfg.Debug.Directory {
t.Fatal("state roots were coupled")
}
}
func TestFileConfigCheckpointEnabledCanBeExplicitlyDisabled(t *testing.T) {
cfg := applyFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: true\n")
if !cfg.Cache.Checkpoints.Enabled || !cloneConfig(cfg).Cache.Checkpoints.Enabled {
t.Fatalf("enabled checkpoint config was not retained: %#v", cfg.Cache.Checkpoints)
}
file := parseFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: false\n")
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if cfg.Cache.Checkpoints.Enabled {
t.Fatalf("explicit false checkpoint config was not applied: %#v", cfg.Cache.Checkpoints)
}
}
func TestFileConfigRejectsTrimmedKeyCollisions(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{
name: "pipeline ids",
yaml: "version: 4\npipelines:\n main: {}\n ' main ': {}\n",
want: "pipeline id \"main\" is duplicated after trimming",
},
{
name: "lane ids",
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells: {}\n ' spells ': {}\n",
want: "artifact lane id \"spells\" is duplicated after trimming",
},
{
name: "reference slots",
yaml: "version: 4\npipelines:\n main:\n references:\n slot: ./one.txt\n ' slot ': ./two.txt\n",
want: "reference slot \"slot\" is duplicated after trimming",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, tt.yaml)
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ApplyFileConfig() error = %v, want context %q", err, tt.want)
}
})
}
}
func TestFileConfigParsesOrderedStepsAndReferenceSources(t *testing.T) {
file := parseFileConfig(t, `version: 4
pipelines:
session:
input: seriatim
steps:
- id: identify-npcs
artifacts:
npc_registry:
extract: dnd/npc-registry
- id: grounded-events
references:
npc_registry:
artifact:
step: identify-npcs
lane: npc_registry
artifacts:
spells:
extract: dnd/spells
`)
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
profile := cfg.Pipelines["session"]
if len(profile.Steps) != 2 || profile.Steps[0].ID != "identify-npcs" || profile.Steps[1].ID != "grounded-events" {
t.Fatalf("steps = %#v", profile.Steps)
}
source := profile.Steps[1].References["npc_registry"]
if source.Artifact == nil || source.Artifact.Step != "identify-npcs" || source.Artifact.Lane != "npc_registry" {
t.Fatalf("generated source = %#v", source)
}
}
func TestFileConfigRejectsAmbiguousReferenceSourceForms(t *testing.T) {
for _, source := range []string{
"artifact: {step: a, lane: b, extra: c}",
"artifact: {step: 1, lane: b}",
"1",
} {
_, err := ParseFileConfigYAML([]byte("version: 4\npipelines:\n p:\n input: text\n references:\n slot: " + source + "\n"))
if err == nil {
t.Fatalf("ParseFileConfigYAML(%q) error = nil", source)
}
}
}
func TestFileConfigRejectsEmptyAndAmbiguousPipelineShapes(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{
name: "empty steps",
yaml: "version: 4\npipelines:\n p:\n input: text\n steps: []\n",
want: "at least one ordered step",
},
{
name: "both forms",
yaml: "version: 4\npipelines:\n p:\n input: text\n artifacts: {}\n steps: []\n",
want: "both artifacts and steps",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, tt.yaml)
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("ApplyFileConfig() error = %v, want context %q", err, tt.want)
}
})
}
}
func TestLoadFileConfigReportsPathAndOperationContext(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.yml")
_, err := LoadFileConfig(missing)
if err == nil || !strings.Contains(err.Error(), "read config file") || !strings.Contains(err.Error(), missing) {
t.Fatalf("missing-file error = %v", err)
}
malformed := filepath.Join(dir, "malformed.yml")
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
t.Fatal(err)
}
_, err = LoadFileConfig(malformed)
if err == nil || !strings.Contains(err.Error(), "parse config file") || !strings.Contains(err.Error(), malformed) {
t.Fatalf("malformed-file error = %v", err)
}
}
func parseFileConfig(t *testing.T, source string) FileConfig {
t.Helper()
file, err := ParseFileConfigYAML([]byte(source))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
return file
}
func applyFileConfig(t *testing.T, source string) Config {
t.Helper()
cfg := Default()
if err := cfg.ApplyFileConfig(parseFileConfig(t, source)); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
return cfg
}

Some files were not shown because too many files have changed in this diff Show More