Compare commits
52 Commits
916d9210fd
...
v0.6.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 77ae132e56 | |||
| deebc89255 | |||
| f208dbe954 | |||
| 7e626753bf | |||
| 917d150279 | |||
| 4da9360d74 | |||
| 5cab4e512e | |||
| b178f1c684 | |||
| a2610757cd | |||
| 3ad34956c5 | |||
| 22e6caa2a0 | |||
| 9cb7462800 | |||
| 87385b7e14 | |||
| 0ae5ea7637 | |||
| 6bc883dfb6 | |||
| fcfff3ad15 | |||
| 0f2b23dce1 | |||
| 61436d7c18 | |||
| 079d5af337 | |||
| 1025001f20 | |||
| da14924a02 | |||
| 2065a8288b | |||
| af0119cc1d | |||
| 54de2b816a | |||
| 4dbbf68051 | |||
| 480680b257 | |||
| 6a1fd7bdb6 | |||
| ccba2ce3f9 | |||
| 1f1967c8d2 | |||
| 5175cb0722 | |||
| ba569594a1 | |||
| acb04954eb | |||
| 610dd3d7c3 | |||
| e92bcfa74c | |||
| e6b2ae88d2 | |||
| b2e83bd6e7 | |||
| 7f01c3e79e | |||
| 7b5f4ebd42 | |||
| 4bb4582695 | |||
| 7d3434e5f6 | |||
| 29872b2e28 | |||
| b487d93186 | |||
| 8dd7a4324d | |||
| 04ba87e174 | |||
| a26d6ed042 | |||
| 759d32403f | |||
| 1a7b20c766 | |||
| 85a5b52be7 | |||
| 9d0faabf61 | |||
| 1c3da3e869 | |||
| 9abd93502f | |||
| c8a29a5fa2 |
@@ -16,7 +16,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"turn_kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["turn", "reaction", "legendary_action", "lair_action", "other"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
"required": ["name", "kind", "source_refs"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"kind": {"type": "string"},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": ["engaged", "killed", "fled", "captured", "incapacitated"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
||||
@@ -18,12 +18,20 @@ when the transcript explicitly describes it being physically destroyed or
|
||||
expended as a non-payment component. Use `transferred` only when possession
|
||||
moves between two distinct named party members.
|
||||
|
||||
Return both `from` and `to` for every occurrence, using `null` when a holder does not
|
||||
apply. For `discovered`, set both holders to `null`. For `acquired`, set `from`
|
||||
to `null` and provide `to`; for `lost` and `consumed`, provide `from` and set
|
||||
`to` to `null`; and for `transferred`, provide both holders. Use `party` only
|
||||
for collective or unresolved party possession, never for either side of a
|
||||
transfer. Do not emit a transfer for a gift, sale, or payment outside the party.
|
||||
Return both `from` and `to` for every occurrence. Use JSON `null`, not an empty
|
||||
string, whenever a holder does not apply. Follow this holder matrix exactly:
|
||||
|
||||
| `kind` | required `from` | required `to` |
|
||||
| --- | --- | --- |
|
||||
| `discovered` | `null` | `null` |
|
||||
| `acquired` | `null` | `party` or the named party member gaining possession |
|
||||
| `lost` | `party` or the named party member losing possession | `null` |
|
||||
| `consumed` | `party` or the named party member consuming the item | `null` |
|
||||
| `transferred` | one named party member | a different named party member |
|
||||
|
||||
Use `party` only for collective or unresolved party possession, never for
|
||||
either side of a transfer. Do not emit a transfer for a gift, sale, or payment
|
||||
outside the party.
|
||||
|
||||
Ordinary non-depleting use is not an occurrence. Do not infer acquisition from a
|
||||
discovery, or discovery from an acquisition: emit both only when each is
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
"required": ["name", "kind", "quantity", "from", "to", "source_refs"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"kind": {"type": "string"},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": ["discovered", "acquired", "lost", "consumed", "transferred"]
|
||||
},
|
||||
"quantity": {"type": ["integer", "null"]},
|
||||
"from": {"type": ["string", "null"]},
|
||||
"to": {"type": ["string", "null"]},
|
||||
|
||||
@@ -5,5 +5,5 @@ evidence, similar objects, or a shared owner as sufficient.
|
||||
Keep currency denominations and materially different item types separate. Keep
|
||||
uncertain aliases separate. Do not infer an item property or uniqueness.
|
||||
|
||||
When selecting a canonical display name, choose one supplied candidate name
|
||||
that is the clearest established designation.
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the clearest established designation.
|
||||
|
||||
@@ -5,4 +5,5 @@ nearby evidence, nested places, or generic labels as sufficient.
|
||||
Keep parent and child places separate, as well as similarly named places and
|
||||
uncertain aliases.
|
||||
|
||||
When selecting a canonical display name, prefer the clearest established name.
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the clearest established name.
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
|
||||
@@ -3,7 +3,8 @@ contextual labels and cited transcript windows. Preserve distinct individuals
|
||||
even when their names are similar or their contextual descriptions are
|
||||
identical.
|
||||
|
||||
When selecting a canonical display name, prefer a complete, stable proper name
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the preferred canonical display name. Prefer a complete, stable proper name
|
||||
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
||||
contextual class, role, title, or relationship descriptor unless the transcript
|
||||
establishes the descriptor as part of the person's name. A longer display name
|
||||
|
||||
@@ -5,9 +5,7 @@ into multiple scenes or use facts that are not supported by it.
|
||||
Return one kind, one concise title, and one concise summary. Choose exactly one
|
||||
kind:
|
||||
|
||||
- combat: active combat materially organizes the scene, including
|
||||
initiative-like exchanges or sustained hostile action. Planning a fight or
|
||||
discussing a completed fight is not combat by itself.
|
||||
- combat: a scene classified as combat under the shared combat policy.
|
||||
- narrative: current-session in-world play that is not principally active
|
||||
combat, a prior-session recap, or sustained out-of-character session
|
||||
discussion. This includes exploration, travel, dialogue, investigation,
|
||||
@@ -20,15 +18,13 @@ kind:
|
||||
play.
|
||||
|
||||
Narrative is the default for actual current-session gameplay that does not meet
|
||||
another definition. When the accepted chunk is mixed:
|
||||
another definition. When the accepted chunk has no substantive active combat:
|
||||
|
||||
1. use combat when active combat is a substantive central activity, even with
|
||||
brief setup, rules clarification, or immediate aftermath;
|
||||
2. otherwise use recap when recounting a previous session is the chunk's
|
||||
1. use recap when recounting a previous session is the chunk's
|
||||
primary table purpose;
|
||||
3. otherwise use meta when sustained out-of-character session discussion is
|
||||
2. otherwise use meta when sustained out-of-character session discussion is
|
||||
primary and in-world progression is no more than incidental; and
|
||||
4. use narrative for all remaining current-session in-world play.
|
||||
3. use narrative for all remaining current-session in-world play.
|
||||
|
||||
Brief table talk, dice resolution, rules clarification, jokes, or
|
||||
administrative comments do not make a gameplay scene meta. A short recollection
|
||||
|
||||
@@ -27,6 +27,8 @@ messages:
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
"required": ["kind", "title", "summary"],
|
||||
"properties": {
|
||||
"kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["combat", "narrative", "recap", "meta"]
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
Classify only whether substantive active combat occurs in the supplied
|
||||
transcript chunk under the shared combat policy. Do not judge the scene title,
|
||||
summary, non-combat subtype, scene boundary, or any other aspect of a scene
|
||||
description.
|
||||
|
||||
Return `combat` when the chunk contains substantive active combat and
|
||||
`non_combat` otherwise. Give a concise, transcript-grounded explanation for
|
||||
the classification.
|
||||
@@ -0,0 +1,23 @@
|
||||
id: dnd.scene_descriptions.validate_combat
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||
repair_attempts: 1
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.scene_descriptions.combat_semantics.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["classification", "explanation"],
|
||||
"properties": {
|
||||
"classification": {
|
||||
"type": "string",
|
||||
"enum": ["combat", "non_combat"]
|
||||
},
|
||||
"explanation": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 512
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
Transcript units are the only evidence for extracted events and factual claims.
|
||||
Every reported factual claim must be supported by cited transcript units. Use
|
||||
integer `start_unit_id` and `end_unit_id` values from the transcript.
|
||||
Within each range, `start_unit_id` must identify the earlier transcript unit and
|
||||
`end_unit_id` the same or a later unit according to transcript order.
|
||||
|
||||
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
||||
than a broad range that bridges unrelated conversation.
|
||||
|
||||
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
@@ -0,0 +1,13 @@
|
||||
Use `combat` only when substantive active combat materially organizes the
|
||||
scene. Active combat includes initiative or turn exchanges, attacks, combat
|
||||
spells, damage, saves, movement, or similarly sustained hostile action.
|
||||
|
||||
Do not use `combat` for planning or preparing for a possible fight; threats,
|
||||
hostile dialogue, or a tense confrontation; immediate aftermath, looting,
|
||||
healing, or discussion of a completed fight; a recap or in-world recollection
|
||||
of earlier combat; or out-of-character rules discussion without active
|
||||
encounter play.
|
||||
|
||||
When a chunk contains substantive active combat alongside brief setup, rules
|
||||
clarification, interruption, phase transition, or immediate aftermath, classify
|
||||
it as `combat`.
|
||||
@@ -1,3 +1,3 @@
|
||||
Candidate material:
|
||||
Candidate material, including the exact valid candidate-number range:
|
||||
|
||||
{{ input "candidates" }}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
Use only the positive integer `candidate_id` values supplied in the candidate material.
|
||||
Use only the positive integer `candidate_number` values supplied in the candidate material. Valid candidate numbers are exactly the inclusive `first` through `last` values declared in `candidate_number_range`; use the explicit number attached to each candidate.
|
||||
|
||||
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group.
|
||||
Transcript unit `id` values and evidence `start_unit_id` and `end_unit_id` values identify source positions. They are not candidate numbers and must never be used in `candidate_numbers` or `canonical_candidate_number`.
|
||||
|
||||
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid.
|
||||
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate numbers, and its `canonical_candidate_number` must be one of those numbers. A candidate may appear in at most one group.
|
||||
|
||||
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer a number that is not explicitly supplied. An empty `duplicate_groups` array is valid.
|
||||
|
||||
The response must conform exactly to the selected JSON schema. Return candidate numbers only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["candidate_ids", "canonical_candidate_id"],
|
||||
"required": ["candidate_numbers", "canonical_candidate_number"],
|
||||
"properties": {
|
||||
"candidate_ids": {
|
||||
"candidate_numbers": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"items": {
|
||||
@@ -21,7 +21,7 @@
|
||||
"minimum": 1
|
||||
}
|
||||
},
|
||||
"canonical_candidate_id": {
|
||||
"canonical_candidate_number": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
|
||||
@@ -83,9 +83,7 @@ entity identity, checkpoint contracts, or cross-request correlation. Changes
|
||||
to shared protocol and policy assets must participate in the normal prompt,
|
||||
schema, and checkpoint fingerprint mechanisms.
|
||||
|
||||
Acceptance of this decision does not imply that the shared mechanism or its
|
||||
consumer migrations are implemented. The
|
||||
[feature roadmap](../roadmap/semantic-reconciliation.md) owns target behavior
|
||||
and status, and the
|
||||
[implementation plan](../roadmap/implementation.md) owns delivery sequence
|
||||
until the work is complete.
|
||||
The shared mechanism and its initial D&D registry consumers are now
|
||||
implemented. Current behavior is documented in
|
||||
[Module Internals](../internal/modules.md#semantic-reconciliation) and
|
||||
[D&D Module Internals](../internal/dnd.md#semantic-registry-reconciliation).
|
||||
|
||||
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
59
docs/adr/0014-feedback-aware-validation-retries.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# ADR-0014: Use feedback-aware validation retries
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-08-26
|
||||
|
||||
## Context
|
||||
|
||||
Validation can identify a candidate defect after a producer has returned an
|
||||
otherwise well-formed result. Retrying without the validator's deterministic,
|
||||
bounded feedback wastes the useful diagnosis, while treating validator
|
||||
execution failures as defects would ask a producer to repair conditions it
|
||||
cannot control. The mechanism must preserve typed producer ownership,
|
||||
checkpoint safety, and the repository's sensitive-data boundaries.
|
||||
|
||||
## Decision
|
||||
|
||||
The implementation will keep three independent budgets: the producer binding's
|
||||
outer `retries` budget, PromptKit's structured-output repair budget, and each
|
||||
validator's execution-retry budget. Validators will run sequentially in their
|
||||
configured order and aggregate both rejections and execution failures before a
|
||||
candidate disposition is selected.
|
||||
|
||||
A correction-capable producer will provide the exact single LLM response that
|
||||
controlled its candidate using the `single_response_v1` protocol. A correction
|
||||
attempt will reconstruct the ordinary request and append exactly two fresh
|
||||
messages: that latest response as `assistant`, followed by one deterministic
|
||||
aggregate correction request as `user`. Earlier turns will not accumulate.
|
||||
|
||||
Validator failures will not recurse into correction. Pipeline policy owns
|
||||
terminal disposition, with field-by-field producer overrides over pipeline
|
||||
defaults: structural failure and semantic rejection default to `fail_run`, and
|
||||
validator execution failure defaults to `warn_continue`. Validators can report
|
||||
facts and bounded corrective guidance, but never decide disposition.
|
||||
|
||||
Rejected and structurally invalid candidates will not advance. A candidate
|
||||
allowed through after a validator execution failure will retain explicit
|
||||
incomplete-validation provenance and will not be checkpointed. Exact response
|
||||
and correction text remain attempt-local: they are excluded from ordinary
|
||||
errors, warnings, manifests, receipts, caches, checkpoints, and default debug
|
||||
summaries.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Retry every producer after any validation outcome. This conflates producer
|
||||
defects with validator operational failures and wastes retry budget.
|
||||
- Let validators decide whether to continue. This would distribute pipeline
|
||||
disposition policy across validators and undermine consistent defaults.
|
||||
- Reuse the full prior conversation. Accumulated turns introduce unbounded
|
||||
prompt growth and make correction behavior depend on incidental history.
|
||||
- Persist raw responses to simplify diagnosis. Raw model output and correction
|
||||
guidance may be sensitive and do not belong in durable pipeline records.
|
||||
|
||||
## Consequences
|
||||
|
||||
The framework gains transport-neutral correction and candidate contracts,
|
||||
producer capability checks, policy resolution, aggregated validation outcomes,
|
||||
and conservative checkpoint handling. Prompt construction remains inside the
|
||||
LLM adapter, while modules remain responsible for accurately exposing the
|
||||
single response that directly controlled a candidate.
|
||||
@@ -0,0 +1,67 @@
|
||||
# ADR-0015: Separate process warnings from quality diagnostics
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-08-27
|
||||
|
||||
## Context
|
||||
|
||||
Notarius currently represents process degradation, incomplete validation,
|
||||
extraction-quality doubt, and routine normalization with one flat warning
|
||||
record. That makes ordinary successful runs noisy, loses the framework context
|
||||
needed to explain a finding, and gives `warning_count` no stable operational
|
||||
meaning. It also permits output encoders to add a warning after the durable
|
||||
warning file has already been written.
|
||||
|
||||
The application needs one bounded diagnostic model that preserves exact
|
||||
occurrence counts while retaining only safe, representative samples. Fresh and
|
||||
resumed logical runs must present the same groups. The model must not alter
|
||||
validation decisions, retry budgets, rejected-output behavior, or process exit
|
||||
policy.
|
||||
|
||||
## Decision
|
||||
|
||||
Warnings are reserved for a completed run that advanced under an allowed
|
||||
process-level degradation or incomplete-work policy. Extraction-quality signals
|
||||
are advisories, and routine accepted transformations are observations. A
|
||||
non-degraded successful run therefore has zero actionable warnings.
|
||||
|
||||
Modules and validators own a diagnostic's disposition, category, reason code,
|
||||
scope, and safe message. The framework adds pipeline origin, including stage,
|
||||
step, lane, module, validator, and chunk context where applicable. It then
|
||||
aggregates deterministically by disposition, category, reason code, and full
|
||||
origin. Chunk context remains on representative samples so equivalent findings
|
||||
across chunks aggregate together.
|
||||
|
||||
Diagnostics carry exact occurrence counts, at most three distinct samples, and
|
||||
numeric omitted-sample metadata. Producers and validators are bounded to 64
|
||||
local groups. Final actionable warning groups are bounded without truncation;
|
||||
the non-warning collection may truncate represented groups while preserving an
|
||||
exact total occurrence count and explicit truncation metadata.
|
||||
|
||||
The public contracts will be versioned: grouped actionable warnings use
|
||||
`notarius.warnings.v2`, grouped advisories and observations use
|
||||
`notarius.diagnostics.v1`, and the run receipt uses
|
||||
`notarius.run-result.v2`. Successful output encoders return logical files or
|
||||
an error; they do not add post-encoding warnings.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Keep one warning list and filter only CLI output. This would leave durable
|
||||
consumers with the same semantically mixed, unbounded contract.
|
||||
- Map reason codes to severity in a central framework registry. This would
|
||||
split module-owned meaning between synchronized policy tables and make new
|
||||
diagnostic meaning implicit.
|
||||
- Preserve local omission warning records. They inflate visible group counts
|
||||
and lose exact occurrence semantics.
|
||||
- Keep output-encoder warnings. A one-pass encoder cannot include those
|
||||
records consistently in files it has already serialized; a two-phase encoder
|
||||
protocol is deferred until a demonstrated need exists.
|
||||
|
||||
## Consequences
|
||||
|
||||
The framework gains validated diagnostic primitives, local collection,
|
||||
origin-aware aggregation, and versioned durable presentation. Existing warning
|
||||
transport remains temporarily while producers migrate. Current architecture,
|
||||
operator, integration, and internal documentation will describe the behavior
|
||||
only as each implementation step lands; this accepted decision does not claim
|
||||
that the migration is complete.
|
||||
76
docs/adr/0016-feedback-aware-module-requested-retries.md
Normal file
76
docs/adr/0016-feedback-aware-module-requested-retries.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# ADR-0016: Use feedback-aware module-requested retries
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-08-28
|
||||
|
||||
## Context
|
||||
|
||||
An LLM-backed module can discover that a structurally valid model response is
|
||||
unsafe while translating it into a typed candidate, before the ordinary
|
||||
validator chain runs. Semantic registry reconciliation is the first such case:
|
||||
the shared core can identify invalid duplicate-group proposals, and typed
|
||||
application can reject a domain-incompatible group while preserving a safe
|
||||
partial result. Repeating the original request without the rejected response or
|
||||
corrective explanation gives the model no information with which to improve.
|
||||
|
||||
The existing feedback-aware validation mechanism already establishes the safe
|
||||
correction protocol, but module-owned retry directives also carry internal
|
||||
reason codes, operator messages, and fallback diagnostics. Those values are not
|
||||
model instructions. Some module retry conditions, including exhausted
|
||||
structured-output repair, also have no valid exact response to append.
|
||||
|
||||
## Decision
|
||||
|
||||
An LLM-backed normalizer may attach optional, bounded model-facing correction
|
||||
guidance to a module-requested retry. Guidance is a separate contract field; the
|
||||
framework never derives it from a reason code, operator message, diagnostic, or
|
||||
error. A feedback-capable directive must include the exact model candidate that
|
||||
controlled the safe fallback and must use `single_response_v1`.
|
||||
|
||||
The artifact-neutral producer-attempt state machine constructs the correction
|
||||
from that exact latest response and the supplied guidance. The producer rebuilds
|
||||
its complete ordinary request and appends the response as one assistant message
|
||||
and the guidance as one user message. Earlier attempts do not accumulate, and
|
||||
the attempt consumes the same configured stage retry budget as operational,
|
||||
structural, validation, and feedback-free module retries.
|
||||
|
||||
A module retry without guidance remains valid and starts a fresh attempt. This
|
||||
is the required behavior when no valid model candidate exists, including after
|
||||
structured-output repair is exhausted. If feedback is supplied without a valid
|
||||
supported candidate, the framework fails the module contract rather than
|
||||
retrying blindly or inventing guidance.
|
||||
|
||||
After retry exhaustion, the normalizer's safe fallback continues through its
|
||||
validator chain. Safe semantic groups may remain applied, unsafe groups remain
|
||||
separate, and bounded fallback diagnostics may explain the process degradation.
|
||||
Exact responses and correction text remain attempt-local and are excluded from
|
||||
ordinary errors, warnings, manifests, receipts, caches, and checkpoints.
|
||||
|
||||
This decision extends, rather than supersedes,
|
||||
[ADR-0014](0014-feedback-aware-validation-retries.md): both correction sources
|
||||
use the same transport-neutral payload, replacement-request semantics, outer
|
||||
retry budget, and sensitive-data boundary.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Continue blind module retries. This preserves a smaller contract but wastes
|
||||
the module's deterministic diagnosis and commonly repeats the same defect.
|
||||
- Convert module safety checks into validators. Typed reconciliation must apply
|
||||
only safe proposal groups and retain a fallback before validation; moving
|
||||
artifact-owned translation and application policy into validators would blur
|
||||
stage ownership.
|
||||
- Copy the retry reason or operator message into the model request. Those values
|
||||
are written for provenance and humans, can contain opaque internal labels,
|
||||
and do not reliably describe a correct replacement.
|
||||
- Require feedback for every module retry. Structural failures may have no
|
||||
valid exact candidate, so this would either prevent useful fresh retries or
|
||||
fabricate prior-response material.
|
||||
|
||||
## Consequences
|
||||
|
||||
The normalize retry contract and generic producer-attempt directive gain an
|
||||
optional correction-guidance field and candidate-pairing validation. Modules
|
||||
that use it must provide semantically meaningful bounded prose and an exact
|
||||
candidate. Registry reconciliation maintains separate operator and model
|
||||
renderers, and policy fingerprints change so checkpoints created under blind
|
||||
retry behavior are not reused.
|
||||
53
docs/cli.md
53
docs/cli.md
@@ -55,8 +55,8 @@ pipeline ID and **--input** are required.
|
||||
| **--session-id id** | Override the generated prompt session identifier with a non-empty value for LLM-backed module calls. |
|
||||
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
|
||||
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
|
||||
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. |
|
||||
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. |
|
||||
| **--reference selector=path** | Add or replace external file reference bindings at pipeline, lane, chunk, or binding scope. Repeatable. |
|
||||
| **--without-reference selector** | Remove matching configured external reference bindings. Repeatable. |
|
||||
|
||||
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
|
||||
**--debug-dir**, **--output-dir**, **--session-id**, and
|
||||
@@ -81,33 +81,62 @@ guidance.
|
||||
|
||||
### Reference selectors
|
||||
|
||||
Use **--reference** only for a reference slot declared by the selected
|
||||
configured target. The accepted selector forms are:
|
||||
Use **--reference** only for reference slots declared by the selected
|
||||
configured targets. Qualification narrows the scope of an override:
|
||||
|
||||
| Form | Target |
|
||||
| --- | --- |
|
||||
| slot=path | The unique selected target that declares slot. |
|
||||
| slot=path | Every selected target that declares slot. |
|
||||
| chunk.slot=path | The chunker. |
|
||||
| merge.slot=path | The unique selected merger that declares slot. |
|
||||
| lane.slot=path | The unique extractor, merger, or normalizer in lane that declares slot. |
|
||||
| lane.slot=path | Every extractor, merger, or normalizer in lane that declares slot. |
|
||||
| lane.extract.slot=path | The extractor in lane. |
|
||||
| lane.merge.slot=path | The merger in lane. |
|
||||
| lane.normalize.slot=path | The normalizer in lane. |
|
||||
|
||||
**--without-reference** uses the same selector forms without =path. Slot
|
||||
names, requiredness, and configured bindings are part of the
|
||||
Pipeline- and lane-scoped selectors are expected to match multiple targets and
|
||||
fail if they match none. A stage-specific selector fails when its lane is not
|
||||
selected or its target does not declare the slot. There is no stage-wide
|
||||
`merge.slot` shorthand; name the lane when targeting a merger.
|
||||
|
||||
CLI bindings override configured external paths. For overlapping CLI
|
||||
selectors, a binding-specific or chunk selector overrides a lane selector, and
|
||||
a lane selector overrides a pipeline selector. The last occurrence wins at
|
||||
equal scope. Binding and unbinding the same concrete target at equal scope is
|
||||
an error; a narrower bind or unbind may create an intentional exception to a
|
||||
broader action.
|
||||
|
||||
**--without-reference** uses the same selector forms without `=path` and
|
||||
removes external bindings only. Neither flag replaces or removes a generated
|
||||
artifact handoff; an external/generated collision is a resolution error.
|
||||
Required slots are checked after all effective changes. CLI reference paths
|
||||
are resolved relative to the process working directory, so subprocess and
|
||||
service callers should use absolute paths. Slot names, accepted media types,
|
||||
size limits, requiredness, and configured generated bindings are part of the
|
||||
[configuration contract](config.md).
|
||||
|
||||
For example, one shared campaign reference can reach every compatible target,
|
||||
with an optional lane-specific exception:
|
||||
|
||||
~~~
|
||||
notarius run dnd-session \
|
||||
--input /data/transcript.json \
|
||||
--reference party=/data/references/party.txt \
|
||||
--reference npc-registry.party=/data/references/npc-party-context.txt
|
||||
~~~
|
||||
|
||||
### Run output
|
||||
|
||||
Without **--json**, standard output contains the completed pipeline ID, counts
|
||||
of normalized and rejected outputs, and the output directory. A debug-enabled
|
||||
run also prints its debug-bundle path to standard output. A successful run with
|
||||
warnings reports the warning count to standard error. The published JSON bundle
|
||||
actionable process warnings reports their group and occurrence counts to
|
||||
standard error. When the selected output module publishes `warnings.json`, the
|
||||
summary also reports that durable file's path. Advisory and observation findings
|
||||
do not produce a warning line. The published JSON bundle
|
||||
is defined by the [JSON output contract](integrations/json-output.md).
|
||||
|
||||
With **--json**, successful standard output is exactly one
|
||||
`notarius.run-result.v1` JSON document followed by a newline, with no
|
||||
`notarius.run-result.v2` JSON document followed by a newline, with no
|
||||
human-oriented status or debug-path line. Its fields and compatibility policy
|
||||
are defined by the [run-result contract](integrations/run-result.md). A caller
|
||||
must check for exit status 0 before decoding this output; a failed write can
|
||||
@@ -171,7 +200,7 @@ go run ./cmd/notarius pipelines list \
|
||||
Successful commands write their primary result to standard output. Warnings and
|
||||
errors are written to standard error.
|
||||
|
||||
For **run --json**, warnings remain on standard error and standard output is a
|
||||
For **run --json**, actionable process warnings remain on standard error and standard output is a
|
||||
machine-readable success result only. Syntax and runtime diagnostics remain on
|
||||
standard error. Parse the result only after the process exits with status 0.
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ model: example-model
|
||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||
its credential environment variable through `api_key_env`; set that variable
|
||||
only in the run environment. PromptKit owns the
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md),
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
|
||||
including `base_profile` inheritance. Notarius passes profiles through without
|
||||
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||
@@ -223,6 +223,7 @@ pipelines:
|
||||
| --- | --- | --- | --- |
|
||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
|
||||
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
|
||||
| **input** | module binding | none | Required. |
|
||||
| **chunk** | module binding | **generic** | Optional. |
|
||||
| **output** | module binding | **json** | Optional. |
|
||||
@@ -251,6 +252,33 @@ explicit `null` and non-integer values are invalid. An explicit value on a
|
||||
deterministic binding or validator is invalid, while a pipeline value simply
|
||||
does not apply to deterministic selections.
|
||||
|
||||
`validation_policy` controls terminal disposition for one complete producer
|
||||
attempt and validator chain. It may appear on a pipeline or a **chunk**,
|
||||
**extract**, **merge**, or **normalize** module binding; input, output, and
|
||||
validator bindings reject it. Every field is optional and resolves in binding,
|
||||
pipeline, then application-default order:
|
||||
|
||||
| Field | Values | Default |
|
||||
| --- | --- | --- |
|
||||
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
|
||||
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
|
||||
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
|
||||
|
||||
The policy object and its fields must be non-null, and unknown fields are
|
||||
rejected. A deterministic producer may not explicitly set
|
||||
**producer_structural_failure** on its binding, although a pipeline-level
|
||||
default remains valid for pipelines that include LLM-backed producers.
|
||||
|
||||
After the producer binding's retry budget is exhausted, an invalid structured
|
||||
response uses **producer_structural_failure**. One or more semantic validator
|
||||
rejections use **semantic_rejection**; rejection takes precedence over an
|
||||
exhausted validator failure or skip. With no rejection, an exhausted validator
|
||||
failure or skip uses **validator_failure**. `reject_output` records the
|
||||
terminal rejection without advancing that candidate. `warn_continue` is valid
|
||||
only for validator execution failure: it advances a structurally valid,
|
||||
otherwise unrejected result with incomplete-validation provenance and without
|
||||
making it reusable checkpoint state.
|
||||
|
||||
A lane has these fields:
|
||||
|
||||
| Field | Type | Default | Rules |
|
||||
@@ -289,7 +317,8 @@ extract:
|
||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
|
||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
||||
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
|
||||
| **retries** | integer | 0 | Non-negative additional complete producer attempts for chunk, extract, merge, and normalize bindings. This single budget covers operational errors, invalid structured output, module-requested normalization retry, and semantic correction. |
|
||||
| **options** | object | none | Must satisfy the selected module. |
|
||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||
@@ -297,11 +326,23 @@ extract:
|
||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||
Validator bindings accept only **module**, **llm_profile**,
|
||||
**structured_output_repair_attempts**, and **options**. They reject
|
||||
**references**, **retries**, and nested **validators**. Deterministic validators
|
||||
reject explicit **llm_profile** and **structured_output_repair_attempts**.
|
||||
**structured_output_repair_attempts**, **retries**, and **options**. Their
|
||||
**retries** value is a non-negative additional validator-execution budget and
|
||||
is valid only when the selected validator is LLM-backed. A validator retry
|
||||
rechecks the same immutable candidate; it never regenerates the producer.
|
||||
They reject
|
||||
**validation_policy**, **references**, and nested **validators**. Deterministic
|
||||
validators reject explicit **llm_profile** and
|
||||
**structured_output_repair_attempts**.
|
||||
Deterministic module bindings also reject those explicit fields.
|
||||
|
||||
An LLM-backed chunk, extract, merge, or normalize producer with both a
|
||||
non-empty validator chain and positive **retries** must declare the supported
|
||||
single-response correction capability. Preparation rejects a configuration
|
||||
that could require semantic correction from a producer that cannot provide an
|
||||
exact prior response. A deterministic producer, or an LLM attempt that did
|
||||
not make a model call, cannot consume a semantic retry after rejection.
|
||||
|
||||
The **json** output module accepts optional **include_chunk_map** and
|
||||
**evidence_context** settings:
|
||||
|
||||
@@ -370,10 +411,15 @@ slot. A generated binding supplies one accepted normalized artifact; it does
|
||||
not name a file. A configured generated dependency remains required even when
|
||||
that consumer slot is otherwise optional.
|
||||
|
||||
Pipeline references are defaults. A matching step-local or binding-local
|
||||
external path overrides a pipeline default. Required slots must be bound after
|
||||
these configuration values and any CLI reference overrides are applied.
|
||||
Reference paths in YAML are resolved relative to the configuration file.
|
||||
Pipeline references are configuration defaults. A matching step-local or
|
||||
binding-local external path overrides a pipeline default. CLI reference
|
||||
bindings are then operational overrides of configured external paths; their
|
||||
pipeline, lane, and binding scopes and precedence are defined by the
|
||||
[CLI reference](cli.md#reference-selectors). A CLI file reference cannot
|
||||
replace a configured generated artifact handoff. Required slots must be bound
|
||||
after configuration and CLI reference actions are applied. Reference paths in
|
||||
YAML are resolved relative to the configuration file; CLI reference paths are
|
||||
resolved relative to the process working directory.
|
||||
|
||||
### D&D Reference Slots
|
||||
|
||||
@@ -463,11 +509,33 @@ Available validator keys are:
|
||||
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
|
||||
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
|
||||
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
|
||||
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
|
||||
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
|
||||
|
||||
`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene:
|
||||
|
||||
~~~yaml
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 1
|
||||
validators:
|
||||
- generic/valid_json
|
||||
- extract/dnd/scene-descriptions/shape
|
||||
- extract/dnd/scene-descriptions/source_refs
|
||||
- generic/valid_json_schema
|
||||
- extract/dnd/scene-descriptions/source_relatedness
|
||||
- module: extract/dnd/scene-descriptions/combat_semantics
|
||||
retries: 1
|
||||
~~~
|
||||
|
||||
An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules.
|
||||
|
||||
The validator retry shown above permits one additional execution against the
|
||||
same scene candidate when the LLM-backed validator itself fails; it is separate
|
||||
from both the extractor's producer retry and PromptKit structural repair.
|
||||
|
||||
When no override is configured, production D&D bindings use the following
|
||||
ordered chains. Each row lists extract then normalize; spell chains are the
|
||||
same at both stages.
|
||||
|
||||
@@ -24,7 +24,9 @@ files. Use absolute paths for service and subprocess deployments. In
|
||||
particular, observe these different resolution rules:
|
||||
|
||||
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||
file; and
|
||||
file;
|
||||
- reference paths passed with `--reference` are resolved relative to the
|
||||
Notarius process working directory; and
|
||||
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||
directory.
|
||||
|
||||
@@ -73,9 +75,21 @@ notarius run dnd-session \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||
--output-dir /absolute/path/to/notarius-output \
|
||||
--reference party=/absolute/path/to/references/party.txt \
|
||||
--reference players=/absolute/path/to/references/players.txt \
|
||||
--reference glossary=/absolute/path/to/references/glossary.txt \
|
||||
--reference spell_catalog=/absolute/path/to/references/spells.json \
|
||||
--json
|
||||
```
|
||||
|
||||
Each unqualified reference is pipeline-scoped: Notarius supplies it to every
|
||||
selected D&D target that declares the slot. A deployment may omit an optional
|
||||
reference it does not maintain, and may use the lane- or binding-qualified
|
||||
forms from the [CLI reference](../cli.md#reference-selectors) for an exceptional
|
||||
override. The registry, scene-description, combat-turn, and NPC-occurrence
|
||||
references declared between ordered steps in the complete configuration are
|
||||
generated artifacts. Do not pass those handoffs on the CLI.
|
||||
|
||||
The caller should:
|
||||
|
||||
- capture stdout and stderr separately;
|
||||
@@ -92,7 +106,7 @@ stream and exit-status contract.
|
||||
## Discover The Published Bundle
|
||||
|
||||
Decode the successful stdout document as a supported run-result schema. For
|
||||
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate
|
||||
the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate
|
||||
unknown fields allowed by that version, but reject an unsupported schema
|
||||
version.
|
||||
|
||||
@@ -101,6 +115,12 @@ root. Do not scan the output root for its newest directory, guess a run ID, or
|
||||
construct a bundle path. Resolve `index_file` beneath `output_directory` and
|
||||
reject an absolute logical path or any result that escapes the bundle root.
|
||||
|
||||
The complete configuration uses the application validation defaults. A caller
|
||||
that requires fully validated D&D artifacts must also require receipt
|
||||
`validation_status: approved`; a successful `incomplete` result reflects the
|
||||
configured validator-failure continuation policy and carries its bounded
|
||||
validator provenance in `validation_summaries`.
|
||||
|
||||
Read `index.json` and locate each requested lane in `output_files` by its exact
|
||||
`lane_id`. Do not guess a lane filename. Before decoding a payload:
|
||||
|
||||
@@ -139,7 +159,8 @@ The JSON encoder always publishes these bundle-management files:
|
||||
| `index.json` | Discovery document for lane and pipeline-wide artifacts. |
|
||||
| `manifest.json` | Run provenance and result summaries. |
|
||||
| `rejected.json` | Rejected pipeline outputs. |
|
||||
| `warnings.json` | Accepted-output and run warnings. |
|
||||
| `warnings.json` | Actionable process-degradation warnings. |
|
||||
| `diagnostics.json` | Advisory and observation findings for accepted artifacts. |
|
||||
|
||||
The complete configuration also requests two pipeline-wide artifacts:
|
||||
|
||||
|
||||
@@ -31,13 +31,36 @@ notarius run pipeline-id \
|
||||
```
|
||||
|
||||
Use absolute paths for supplied input, configuration, output-root, and
|
||||
reference files. Notarius generates a stable prompt session for the resolved
|
||||
input module and exact input bytes. Pass **--session-id** only when intentionally
|
||||
grouping different invocations under a different session. Supply credentials
|
||||
through Notarius's documented configuration and environment mechanisms, never
|
||||
as command-line arguments or generated secret-bearing configuration. In
|
||||
particular, a session identifier is provider-visible and is not a credential
|
||||
mechanism.
|
||||
reference files. Pass each external reference as its own argument-vector pair;
|
||||
do not construct and invoke a shell command. An unqualified reference selector
|
||||
supplies that file to every compatible selected target. Lane and stage
|
||||
qualification are available for exceptional overrides, while generated
|
||||
same-run references remain part of configured pipeline composition. The
|
||||
[CLI reference](../cli.md#reference-selectors) owns the exact selector and
|
||||
precedence contract.
|
||||
|
||||
The maintained D&D subprocess workflow uses this facility for campaign context:
|
||||
|
||||
```sh
|
||||
notarius run dnd-session \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||
--output-dir /absolute/path/to/notarius-output \
|
||||
--reference party=/absolute/path/to/references/party.txt \
|
||||
--reference players=/absolute/path/to/references/players.txt \
|
||||
--reference glossary=/absolute/path/to/references/glossary.txt \
|
||||
--reference spell_catalog=/absolute/path/to/references/spells.json \
|
||||
--json
|
||||
```
|
||||
|
||||
Only pass the external references available to and desired by the deployment.
|
||||
Notarius generates a stable prompt session for the resolved input module and
|
||||
exact input bytes; reference changes do not change it. Pass **--session-id**
|
||||
only when intentionally grouping different invocations under a different
|
||||
session. Supply credentials through Notarius's documented configuration and
|
||||
environment mechanisms, never as command-line arguments or generated
|
||||
secret-bearing configuration. In particular, a session identifier is
|
||||
provider-visible and is not a credential mechanism.
|
||||
|
||||
Wait for the process before interpreting standard output. Only an exit status
|
||||
of 0 permits decoding the receipt. On a nonzero exit, retain standard error for
|
||||
@@ -69,10 +92,17 @@ lanes. The caller decides which lane IDs are required for its own work and
|
||||
which are optional; it should make that decision explicitly rather than infer
|
||||
failure from the receipt counts alone.
|
||||
|
||||
When complete validation is required, also require receipt
|
||||
`validation_status: approved` and inspect `validation_summaries`. A successful
|
||||
run with `validation_status: incomplete` contains a structurally valid result
|
||||
that advanced after validator execution could not complete under the configured
|
||||
`warn_continue` policy. It is not reusable checkpoint state and should not be
|
||||
silently treated as fully reviewed by the caller.
|
||||
|
||||
## Preserve Provenance And Handle Data Carefully
|
||||
|
||||
Keep the receipt with the published `manifest.json`, and retain
|
||||
`rejected.json` and `warnings.json` when review or later provenance requires
|
||||
`rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires
|
||||
them. Treat the input, output bundle, cache, debug bundle, and captured process
|
||||
logs as potentially sensitive data. Apply the caller's access controls and
|
||||
retention policy, and avoid copying secrets into arguments, logs, or
|
||||
|
||||
@@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo
|
||||
## Bundle Layout
|
||||
|
||||
All paths below are logical, relative, slash-separated bundle paths. The
|
||||
encoder always emits the first four JSON files below and adds lane or
|
||||
encoder always emits the first five JSON files below and adds lane or
|
||||
pipeline-wide artifact files when their corresponding artifacts are available:
|
||||
|
||||
A subprocess caller first obtains the physical bundle root from the
|
||||
@@ -21,7 +21,8 @@ root for the logical discovery described here.
|
||||
| `index.json` | Entry point that names the other published files and lane payloads. |
|
||||
| `manifest.json` | Run provenance and result summaries. |
|
||||
| `rejected.json` | Rejected pipeline outputs. |
|
||||
| `warnings.json` | Accepted-output and run warnings. |
|
||||
| `warnings.json` | Actionable process-degradation warnings. |
|
||||
| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. |
|
||||
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
||||
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
||||
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
|
||||
@@ -39,7 +40,8 @@ normalized lanes has this valid minimal index:
|
||||
"manifest_file": "manifest.json",
|
||||
"output_files": [],
|
||||
"rejected_file": "rejected.json",
|
||||
"warnings_file": "warnings.json"
|
||||
"warnings_file": "warnings.json",
|
||||
"diagnostics_file": "diagnostics.json"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -49,6 +51,7 @@ normalized lanes has this valid minimal index:
|
||||
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
|
||||
| `rejected_file` | Yes | Always `rejected.json`. |
|
||||
| `warnings_file` | Yes | Always `warnings.json`. |
|
||||
| `diagnostics_file` | Yes | Always `diagnostics.json`. |
|
||||
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
|
||||
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
|
||||
|
||||
@@ -92,7 +95,7 @@ group into the following externally observable summaries:
|
||||
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
||||
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
||||
| Source and references | `source_digests`, `references` |
|
||||
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
|
||||
| Published result summaries | `normalized_outputs`, `rejected_outputs`, `validation_summaries` |
|
||||
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
||||
|
||||
`references` records provenance such as the target, slot, origin, digest,
|
||||
@@ -102,6 +105,16 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is
|
||||
provenance for the plan used by this run; cache records, debug artifacts, and
|
||||
other operational state are not published as bundle files.
|
||||
|
||||
Each `validation_summaries` entry is a bounded outcome for one producer result.
|
||||
It has required `status`, `producer_attempt_count`, and `terminal_action`;
|
||||
the stage and affected step, lane, module, or chunk identity are present when
|
||||
applicable. `status` is `complete`, `rejected`, or `incomplete`.
|
||||
`rejecting_validators`, `reason_codes`, and `incomplete_validators` preserve
|
||||
configured validator order and omit later duplicates. Entries contain no raw
|
||||
candidate response, correction guidance, validator diagnostic message, or
|
||||
artifact payload. The same shape may appear as `validation` on an affected
|
||||
rejection entry.
|
||||
|
||||
When present, `metadata.session_id` is the effective non-secret routing
|
||||
correlation identifier used for the run. It can be visible to providers and is
|
||||
not a substitute for a cache or checkpoint identity. Its generation and
|
||||
@@ -122,16 +135,57 @@ These values describe observed execution; they are not a backend-registration
|
||||
interface. Entries that differ by backend or effective reasoning remain
|
||||
distinct even when their profile, provider, and model are otherwise equal.
|
||||
|
||||
## Rejections And Warnings
|
||||
## Rejections, Warnings, And Diagnostics
|
||||
|
||||
`rejected.json` is always an object with a `rejected` array. Each entry has
|
||||
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
||||
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
||||
`diagnostic_artifact_path` are present only when applicable.
|
||||
`diagnostic_artifact_path` are present only when applicable. An entry may also
|
||||
contain the bounded `validation` summary described above; the existing singular
|
||||
validator and reason fields remain the first configured rejection for
|
||||
compatibility.
|
||||
|
||||
`warnings.json` is always an object with a `warnings` array. Each warning has
|
||||
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
|
||||
there is nothing to report.
|
||||
`warnings.json` is always the `notarius.warnings.v2` envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "notarius.warnings.v2",
|
||||
"group_count": 0,
|
||||
"occurrence_count": 0,
|
||||
"groups": []
|
||||
}
|
||||
```
|
||||
|
||||
It contains only process warnings. `group_count` is exact, and
|
||||
`occurrence_count` is the exact sum of its group occurrence counts.
|
||||
|
||||
`diagnostics.json` is always the `notarius.diagnostics.v1` envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "notarius.diagnostics.v1",
|
||||
"group_count": 0,
|
||||
"occurrence_count": 0,
|
||||
"truncated": false,
|
||||
"unrepresented_occurrence_count": 0,
|
||||
"groups": []
|
||||
}
|
||||
```
|
||||
|
||||
It contains only advisory and observation groups. `group_count` counts groups
|
||||
represented in `groups`; `occurrence_count` includes both represented and
|
||||
unrepresented occurrences. When `truncated` is true,
|
||||
`unrepresented_occurrence_count` is the exact number omitted from group
|
||||
representation.
|
||||
|
||||
Each group has `disposition`, `category`, `reason_code`, framework-owned
|
||||
`origin`, exact `occurrence_count`, bounded `samples`, and
|
||||
`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk
|
||||
ID and zero-based chunk index when applicable. A group retains at most three
|
||||
distinct samples. The framework fails rather than truncating actionable
|
||||
warnings beyond 128 groups; it represents at most 256 advisory/observation
|
||||
groups and records further occurrences through the diagnostic truncation
|
||||
fields above.
|
||||
|
||||
## Compatibility
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# PromptKit Integration
|
||||
|
||||
Notarius pins
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0)
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
|
||||
as its in-process prompt engine. The upstream
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/consumers/pkg-promptkit.md)
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
owns the public engine API, and the upstream
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md)
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
owns prompt, profile, and schema file contracts.
|
||||
|
||||
## Supported Boundary
|
||||
@@ -15,7 +15,8 @@ Notarius relies on the root `promptkit` package to:
|
||||
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
||||
operator and application-fallback profile sources;
|
||||
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
||||
variables, a direct session ID, prompt identity, and profile selection, then
|
||||
variables, a direct session ID, prompt identity, profile selection, and
|
||||
optional appended rendered messages, then
|
||||
record credential-redacted details and run that exact execution;
|
||||
- return rendered debug material, validated structured output, selected
|
||||
profile, backend, effective model metadata, and token usage;
|
||||
@@ -26,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
|
||||
admission exhaustion through `ErrCapacityExceeded`.
|
||||
|
||||
The pinned
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/backends.go)
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
|
||||
owns the registration and backend-capacity contract.
|
||||
|
||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||
@@ -52,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
|
||||
field, which is authoritative for provider session behavior. It also retains
|
||||
the same value as the `session_id` prompt variable for maintained prompt
|
||||
compatibility. The generated identifier is 76 ASCII characters, within
|
||||
PromptKit v0.8.0's 256-code-point session limit. Session IDs are non-secret
|
||||
PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
|
||||
correlation identifiers and may be exposed to providers and provider
|
||||
observability. The CLI contract owns generation and override behavior.
|
||||
|
||||
@@ -85,12 +86,38 @@ configuration and deployment workflow are defined in
|
||||
[Operations](../operations.md#promptkit-profile-deployment).
|
||||
|
||||
PromptKit owns `base_profile` resolution under its
|
||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md).
|
||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||
Notarius records the selected leaf identity and resolved target without parsing
|
||||
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||
reach the provider without authorization, which can result in a 401 or 403.
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
|
||||
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
|
||||
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
||||
Maintained Notarius prompt definitions use only `system` and `user`.
|
||||
|
||||
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
|
||||
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
|
||||
exactly two messages in order: the latest validated producer response with
|
||||
role `assistant`, then deterministic validation guidance with role `user`.
|
||||
It never exposes a general caller-selected role API, accumulates earlier
|
||||
correction turns, or changes the ordinary prompt prefix. Ordinary requests
|
||||
leave appended messages unset.
|
||||
|
||||
PromptKit preserves supplied content but does not own Notarius's correction
|
||||
bounds. Notarius rejects invalid UTF-8, blank, or oversized assistant material
|
||||
(at most 1 MiB), guidance (at most 64 KiB), and combined content (at most
|
||||
1,114,112 bytes) before preparing the request. The transport-neutral
|
||||
application contract owns defensive copying and these limits. Default request
|
||||
and terminal summaries retain only safe counts, digests, identities, and usage;
|
||||
complete appended messages remain limited to the explicitly requested detailed
|
||||
debug trace.
|
||||
|
||||
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
|
||||
profile catalogs from independently versioned transitive modules. Notarius
|
||||
does not import or register either catalog; PromptKit retains catalog source,
|
||||
identity, precedence, credential, and capacity ownership.
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
|
||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||
|
||||
@@ -9,36 +9,58 @@ Command syntax, streams, and exit statuses are defined in the
|
||||
|
||||
## Schema
|
||||
|
||||
The current schema version is `notarius.run-result.v1`.
|
||||
The current schema version is `notarius.run-result.v2`.
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. |
|
||||
| `schema_version` | Yes | Exactly `notarius.run-result.v2`. |
|
||||
| `run_id` | Yes | The finalized Notarius run identifier. |
|
||||
| `pipeline_id` | Yes | The effective pipeline identifier. |
|
||||
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
|
||||
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
|
||||
| `normalized_output_count` | Yes | Number of final normalized outputs. |
|
||||
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
||||
| `warning_count` | Yes | Number of final run warnings. |
|
||||
| `warning_group_count` | Yes | Exact number of actionable warning groups. |
|
||||
| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. |
|
||||
| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. |
|
||||
| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. |
|
||||
| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. |
|
||||
| `validation_status` | Yes | The final run manifest validation status. |
|
||||
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
|
||||
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
||||
|
||||
For the production `json` output module, `index_file` is present only when the
|
||||
completed run returned exactly one logical output file named `index.json`.
|
||||
For another output module, its absence does not indicate a failed run.
|
||||
`validation_status` is `approved`, `rejected`, or `incomplete`; `incomplete`
|
||||
means one or more otherwise accepted results advanced under validator-failure
|
||||
`warn_continue`.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "notarius.run-result.v1",
|
||||
"schema_version": "notarius.run-result.v2",
|
||||
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||
"pipeline_id": "dnd-session",
|
||||
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||
"index_file": "index.json",
|
||||
"normalized_output_count": 6,
|
||||
"rejected_output_count": 2,
|
||||
"warning_count": 1,
|
||||
"validation_status": "rejected"
|
||||
"warning_group_count": 1,
|
||||
"warning_occurrence_count": 2,
|
||||
"diagnostic_group_count": 3,
|
||||
"diagnostic_occurrence_count": 5,
|
||||
"diagnostics_truncated": false,
|
||||
"validation_status": "incomplete",
|
||||
"validation_summaries": [
|
||||
{
|
||||
"stage": "extract",
|
||||
"lane_id": "spells",
|
||||
"status": "incomplete",
|
||||
"incomplete_validators": ["dnd/spells/source_refs"],
|
||||
"producer_attempt_count": 1,
|
||||
"terminal_action": "warn_continue"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@@ -48,8 +70,11 @@ For another output module, its absence does not indicate a failed run.
|
||||
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
||||
`output_directory` is the run-specific bundle, not the configured output root.
|
||||
|
||||
The receipt is a summary and discovery document. It does not contain lane
|
||||
descriptors, payloads, manifest data, rejections, warnings, or file contents.
|
||||
The receipt is a summary and discovery document. Its optional validation
|
||||
summaries contain only stable status, identity, validator names, reason codes,
|
||||
attempt counts, and terminal actions. It does not contain lane descriptors,
|
||||
payloads, manifest payloads, rejection messages, warnings, raw model responses,
|
||||
correction guidance, or file contents.
|
||||
For the production JSON output, resolve `index_file` beneath
|
||||
`output_directory`, reject path escapes, and use the
|
||||
[Published JSON Output contract](json-output.md) to discover logical files and
|
||||
|
||||
@@ -115,6 +115,16 @@ to checkpoint identity and `pipeline.RunInput`. The public flag and stability
|
||||
contract are defined by the [CLI reference](../cli.md#run); framework and LLM
|
||||
packages only transport the supplied value.
|
||||
|
||||
The CLI also owns the scope grammar for reference flags. It enumerates the
|
||||
selected chunk and lane targets from registered module specifications, expands
|
||||
pipeline- and lane-scoped actions into exact stage-and-lane bindings, and
|
||||
resolves overlapping bind and unbind actions by specificity before calling
|
||||
configuration resolution. The generic pipeline therefore receives only exact
|
||||
`ReferenceBinding` and `ReferenceUnbind` values and has no knowledge of CLI
|
||||
selector syntax. Configuration resolution retains ownership of configured
|
||||
external/generated conflicts, required slots, and module compatibility; file
|
||||
materialization still occurs afterward.
|
||||
|
||||
For `run --json`, the CLI constructs and encodes its private run-result receipt
|
||||
after a successful runner result is available, before it publishes logical
|
||||
output files. It writes the prepared receipt to standard output only after
|
||||
@@ -166,8 +176,9 @@ is discoverable.
|
||||
- **internal/cli/production_contract_test.go** covers registrar composition,
|
||||
production catalog contents, assets, and representative configuration
|
||||
validation.
|
||||
- **internal/cli/reference_contract_test.go** covers CLI reference overrides,
|
||||
origin separation, and materialization boundaries.
|
||||
- **internal/cli/reference_contract_test.go** covers scoped CLI reference
|
||||
expansion, specificity, bind/unbind conflicts, generated-reference
|
||||
protection, origin separation, and materialization boundaries.
|
||||
- **internal/cli/state_hardening_test.go** covers safe run identity, state
|
||||
roots, and failure ordering.
|
||||
|
||||
|
||||
@@ -74,10 +74,19 @@ profile-free, and no second inheritance decision occurs during execution. The
|
||||
public field definitions and precedence are owned by
|
||||
[Configuration](../config.md#pipelines).
|
||||
|
||||
The resolver retains configured `validation_policy` overrides and derives one
|
||||
detached concrete terminal policy for the chunk producer and every lane's
|
||||
extract, merge, and normalize producers. That field-by-field inheritance is
|
||||
complete before preparation, and the effective values contribute to pipeline
|
||||
and checkpoint identity; execution does not interpret configuration defaults.
|
||||
|
||||
The framework resolver supplies defaults, selects lanes, resolves validator
|
||||
chains, checks registered module and artifact compatibility, validates module
|
||||
options, and returns the fixed ordered pipeline shape. The resulting
|
||||
**EffectiveConfig** retains the selected ID, requested selection and reference
|
||||
options, and returns the fixed ordered pipeline shape. Positive validator retry
|
||||
budgets require an LLM-backed selected validator; deterministic validators are
|
||||
rejected during resolution. Eligible LLM-backed producer specifications also
|
||||
contribute their declared correction protocol to the resolved metadata. The
|
||||
resulting **EffectiveConfig** retains the selected ID, requested selection and reference
|
||||
changes, a clone of the input configuration, and the resolved pipeline.
|
||||
Callers may therefore retain or modify their input slices and maps without
|
||||
changing the resolved result, and later consumers cannot mutate the original
|
||||
@@ -94,8 +103,8 @@ runtime error class described in the [CLI reference](../cli.md#output-streams-an
|
||||
|
||||
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
||||
after defaults, lane selection, module bindings, reference bindings, validator
|
||||
chains, effective LLM profiles, and artifact schema identity have been
|
||||
resolved. The digest excludes
|
||||
chains, selected correction protocols, effective LLM profiles, and artifact
|
||||
schema identity have been resolved. The digest excludes
|
||||
its own stored value. It identifies resolved composition rather than raw YAML
|
||||
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
||||
provenance before execution; cache and checkpoint identity have additional
|
||||
|
||||
@@ -29,11 +29,17 @@ The D&D registrar registers the family’s artifact codecs, extractors, typed
|
||||
append-order mergers, normalizers, validators, prompt assets, fallback LLM
|
||||
profile asset, and default validator chains. Each extractor and normalizer has
|
||||
a stable module spec, explicit execution class, strict option decoding, and a
|
||||
typed builder. Scene chunking, every extractor, and NPC, location, and item-registry
|
||||
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
||||
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
||||
resolved-pipeline debug data and determines which selected bindings inherit the
|
||||
pipeline profile. Configuration remains the canonical owner of the exact keys,
|
||||
typed builder. Scene chunking, every extractor, and NPC, location, and
|
||||
item-registry normalization are registered as `llm_backed`; the remaining
|
||||
current D&D mergers and normalizers are `deterministic`. The metadata is
|
||||
available to catalog inspection and resolved-pipeline debug data and determines
|
||||
which selected bindings inherit the pipeline profile. The registry normalizers
|
||||
use `single_response_v1`, forwarding corrections to their reconciliation
|
||||
completion and retaining the accepted raw proposal only as an owned model
|
||||
candidate. When deterministic proposal safety or typed application rejects a
|
||||
group, they provide separate model-facing prose for a corrective module retry;
|
||||
internal issue categories, reason codes, and operator messages remain
|
||||
diagnostic-only. Configuration remains the canonical owner of the exact keys,
|
||||
profile precedence, and validator order.
|
||||
|
||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||
@@ -42,6 +48,13 @@ unknown fields, while preserving semantic candidates for deterministic
|
||||
validation. Do not promote a private response envelope into a durable schema;
|
||||
the contracts above define durable data.
|
||||
|
||||
A D&D producer that declares `single_response_v1` forwards any supplied
|
||||
semantic correction to its structured completion and returns an owned copy of
|
||||
that completion's exact validated raw response as its model candidate. It does
|
||||
not serialize normalized artifacts to create that candidate, so deterministic
|
||||
identity, evidence, warning, and durable-schema behavior remains separate from
|
||||
the model transport material.
|
||||
|
||||
## Prompt Construction
|
||||
|
||||
D&D LLM-facing content lives beneath `assets/dnd/`. Each module contributes a
|
||||
@@ -108,14 +121,27 @@ presentation, and final ephemeral generic transcript windows. These orders and
|
||||
cache controls are prompt behavior; change them only through the owning
|
||||
manifest and prompt declaration.
|
||||
|
||||
NPC, item, and location registry reconciliation translate shared proposal
|
||||
safety categories into bounded prose that references only the response-local
|
||||
duplicate-group ordinals and candidate handles. Item reconciliation appends its
|
||||
typed rule that currency may be consolidated only with aliases of the same
|
||||
denomination and never with non-currency items. The next normalize attempt
|
||||
receives that prose with the exact defective proposal under the shared
|
||||
replacement-request protocol. Structurally invalid output has no valid proposal
|
||||
candidate and receives a fresh feedback-free attempt instead. If the stage
|
||||
budget is exhausted, safe groups stay applied, unsafe groups stay separate, and
|
||||
one fallback warning summarizes the final defect without raw model content.
|
||||
|
||||
## Evidence, Candidates, And Normalization
|
||||
|
||||
The current transcript is the only durable evidence source. Extractors assign
|
||||
the current source identity, preserve candidate evidence ranges for validators,
|
||||
and canonically order or remove exact duplicate ranges without asking the
|
||||
model to repair semantic errors. Campaign context and generated artifacts may
|
||||
ground names or control routing, but they never establish evidence for a D&D
|
||||
result.
|
||||
the current source identity and losslessly order any reversed range whose two
|
||||
endpoints resolve in that source, using transcript position rather than numeric
|
||||
unit-ID order. They then canonically order ranges and remove exact duplicates.
|
||||
This routine canonicalization does not request a retry or emit a warning.
|
||||
Unresolvable or wrong-source ranges remain unchanged for validators to reject.
|
||||
Campaign context and generated artifacts may ground names or control routing,
|
||||
but they never establish evidence for a D&D result.
|
||||
|
||||
Default chains keep responsibilities separate: structural validators assess the
|
||||
candidate, source-reference validators resolve cited ranges against the current
|
||||
@@ -125,15 +151,119 @@ relatedness validators report advisory evidence concerns. The configured order
|
||||
is documented in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
|
||||
The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the
|
||||
D&D family's LLM-backed review of only combat versus non-combat classification.
|
||||
It selects the shared combat-policy prompt fragment and asks the model to
|
||||
classify the current chunk independently as `combat` or `non_combat` without
|
||||
receiving the proposed scene kind. Deterministic code compares that
|
||||
classification with the proposed kind and either approves it or produces the
|
||||
appropriate correction guidance. This keeps every schema-valid classification
|
||||
interpretable and avoids anchoring the reviewer on the producer's answer. It
|
||||
does not assess titles, summaries, non-combat subtype, or scene boundaries;
|
||||
deferred boundary-coherence review remains separate. It is opt-in;
|
||||
[Configuration](../config.md) owns selection and retry/failure behavior.
|
||||
|
||||
### Combat-semantics provider evaluation
|
||||
|
||||
The human-reviewed corpus at
|
||||
`internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json`
|
||||
owns the proposed kind, expected combat classification, and reviewer rationale
|
||||
for each synthetic case. Its package test validates the fixture contract only.
|
||||
Provider evaluation remains an explicit maintainer operation and must not be
|
||||
added to the default offline test suite.
|
||||
|
||||
Use the following protocol before proposing default-chain inclusion:
|
||||
|
||||
1. Record the Notarius commit, prompt and schema fingerprints, provider, model,
|
||||
profile settings, reasoning effort, structural-repair setting, number of
|
||||
repetitions, and evaluation date before collecting results. Do not revise
|
||||
expected classifications merely to agree with provider output; a substantive
|
||||
corpus correction requires independent human review.
|
||||
2. Exercise the production validator construction and prompt assets from an
|
||||
explicitly invoked, disposable evaluation driver or test in the validator
|
||||
package. For each corpus case, construct transcript source units from the
|
||||
listed IDs and text, assign matching per-unit source references, and use
|
||||
`source.MaterializeChunkPlan` with one range spanning those units. Construct
|
||||
exactly one scene whose ID and source range match that chunk and whose kind
|
||||
is the case's `proposed_kind`; title and summary may use fixed placeholders
|
||||
because the validator neither receives nor evaluates them. Invoke the typed
|
||||
validator at extract stage through the production registry and scheduled LLM
|
||||
client. Do not commit provider credentials, generated source material, or an
|
||||
always-on live test.
|
||||
3. Compare the model classification with `expected_classification`, then verify
|
||||
that its deterministic comparison with `proposed_kind` yields the expected
|
||||
approval or rejection direction.
|
||||
Record an unexpected approval of an expected rejection as a false
|
||||
acceptance, an unexpected rejection of an expected approval as a false
|
||||
rejection, and any validator execution failure separately from semantic
|
||||
accuracy. Retain per-case results so repeated trials and systematic failure
|
||||
modes remain visible.
|
||||
4. Evaluate producer correction separately with representative complete
|
||||
scene-description runs configured as shown in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
For every initial semantic rejection, record whether the next producer
|
||||
attempt returns the requested combat status and is approved. Do not count a
|
||||
PromptKit structural repair as a producer-correction attempt.
|
||||
5. Run the correction evaluation with debug capture enabled and without reused
|
||||
extraction checkpoints. Record added validator and producer calls, elapsed
|
||||
latency, and cumulative prompt, completion, cached, and total token usage
|
||||
from the debug attempt records. Compare these values with an otherwise
|
||||
identical run whose scene-description chain omits the semantic validator.
|
||||
|
||||
The default-chain review must consider classification error, false acceptance,
|
||||
false rejection, execution failure, producer-correction success, added calls,
|
||||
latency, and token use together. A structurally successful provider run alone
|
||||
is not evidence that the validator should become a default.
|
||||
|
||||
Every producer-correctable D&D rejection describes all currently detectable
|
||||
corrections in transcript-grounded domain terms, using contextual names,
|
||||
model-owned artifact fields, and source segment ranges when useful. Validators
|
||||
collect independent record defects in one pass so one retry does not merely
|
||||
reveal the next issue. Shared D&D diagnostic helpers keep repeated rules and
|
||||
record descriptions stable, de-duplicated, and bounded; each artifact family
|
||||
continues to own the semantic rule and its prose.
|
||||
|
||||
Operator diagnostics and model guidance are separate products of the same
|
||||
assessment. Operator messages may use typed paths, reason details, and opaque
|
||||
application identities. Correction guidance must not copy those messages or
|
||||
ask the model to reproduce durable entity IDs, hashes, validator module keys,
|
||||
reason codes, or Go field paths. A registry-normalization rejection instead
|
||||
speaks in terms of the duplicate-group proposal response the normalizer can
|
||||
actually revise. Normalization-only deterministic invariants retain useful
|
||||
operator detail but do not imply that a model controls derived ordering or
|
||||
identity. Only bounded actionable semantic guidance is eligible for a
|
||||
correction prompt.
|
||||
|
||||
Private LLM schemas use simple enums for closed categorical fields when the
|
||||
provider-compatible shape can express the rule directly. Deterministic typed
|
||||
validators retain the same checks as defense in depth and for non-LLM
|
||||
producers. Private schemas keep every property required and avoid optional
|
||||
properties, `uniqueItems`, and conditional cross-field logic.
|
||||
|
||||
Item-occurrence shape validation groups repeated holder mistakes by occurrence
|
||||
kind and gives the producer the required JSON null/non-null relationship. It
|
||||
identifies affected records by contextual item name and cited transcript range,
|
||||
never by the deterministically attached durable item ID. Holder mistakes remain
|
||||
semantic rejections rather than silent rewrites because changing a holder can
|
||||
also change the meaning of the occurrence kind.
|
||||
|
||||
Enemy-event extraction additionally rejects a second `engaged` observation for
|
||||
the same comparison identity within one scene-scoped result. Normalization may
|
||||
combine results from distinct scenes, so it intentionally does not apply that
|
||||
rule. Configuration owns the exact validator key and chain position.
|
||||
|
||||
Extraction source-reference validators share one full-span chunk-containment
|
||||
policy. After ordinary reference validity succeeds, the policy resolves both
|
||||
endpoints through document order and requires every source unit in the
|
||||
inclusive range to be present in the current chunk. It does not assume numeric
|
||||
unit-ID ordering, mutate input, or weaken wrong-source and unresolved-reference
|
||||
validation. Scene descriptions remain separate because their validator owns an
|
||||
exact one-scene range contract rather than general extraction containment.
|
||||
|
||||
Normalizers are deterministic for spells, combat turns, item occurrences, NPC
|
||||
occurrences, scene descriptions, enemy events, and location occurrences. They
|
||||
canonicalize display values and evidence, use source-document order for stable
|
||||
output, and issue bounded warnings for changes or collapsed duplicates. NPC,
|
||||
output, and emit bounded normalization observations for changes or collapsed duplicates. NPC,
|
||||
item, and location registry normalizers are intentional exceptions: each first
|
||||
produces a deterministic candidate set, then may use a bounded structured-LLM
|
||||
proposal to reconcile identity groups.
|
||||
@@ -142,13 +272,16 @@ proposal to reconcile identity groups.
|
||||
|
||||
The three registry normalizers instantiate the domain-neutral
|
||||
`internal/framework/semanticreconcile` engine with default bounds. Each
|
||||
eligible candidate receives a contiguous, one-based `candidate_id` for that
|
||||
request. The model sees that handle, the candidate label and source-free
|
||||
evidence ranges, plus bounded transcript windows; it returns only duplicate
|
||||
eligible candidate receives a contiguous, one-based `candidate_number` for that
|
||||
request, and candidate material declares the exact inclusive range. The model
|
||||
sees that handle, the candidate label and source-free evidence ranges, plus
|
||||
bounded transcript windows; it returns only duplicate
|
||||
groups of supplied handles and one supplied canonical handle per group. It
|
||||
never returns names, evidence, durable IDs, or replacement records. Identical
|
||||
labels and evidence remain independently selectable because their handles are
|
||||
distinct.
|
||||
distinct. Transcript unit `id` values and evidence `start_unit_id` and
|
||||
`end_unit_id` values are source positions in a separate namespace and are
|
||||
never valid candidate numbers.
|
||||
|
||||
The generic core owns the mandatory handle protocol, candidate and transcript
|
||||
presentation, the private response schema, source-reference validation,
|
||||
|
||||
@@ -42,6 +42,22 @@ observability. The adapter returns PromptKit’s validated raw bytes rather than
|
||||
re-encoding the decoded target. An empty optional material is represented as
|
||||
one space so its named input is retained by PromptKit.
|
||||
|
||||
When a request includes semantic correction material, the adapter validates and
|
||||
defensively copies it before preparation, then appends exactly two messages
|
||||
after the ordinarily rendered prompt: the prior response as an assistant
|
||||
message and the correction guidance as a user message. Requests without a
|
||||
correction do not add messages or introduce caller roles. Ordinary request
|
||||
summaries record correction byte counts and digests only; complete messages are
|
||||
available solely in an explicitly requested debug trace.
|
||||
|
||||
The adapter leaves the ordinary rendered message prefix, named inputs,
|
||||
variables, session, profile, execution overrides, prepared-execution path, and
|
||||
PromptKit repair policy unchanged for a corrected request. It never imports a
|
||||
PromptKit message type into a module or pipeline contract. PromptKit reports
|
||||
actual structural repair count and cumulative token usage per completion; the
|
||||
pipeline's safe terminal debug record projects those values without copying
|
||||
message content.
|
||||
|
||||
Client construction may also receive a run-wide reasoning-effort override from
|
||||
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
||||
creates a fresh PromptKit execution override for each request: a nil pointer
|
||||
@@ -229,10 +245,15 @@ or corrective call remain provider-neutral operational errors with the same
|
||||
redaction boundary.
|
||||
|
||||
Structural repair does not replace pipeline retry behavior: a binding's
|
||||
configured retry count reruns its complete stage attempt after an error or
|
||||
rejection. The pipeline owns attempt lifecycle, validation chains, and retry
|
||||
diagnostics; see [Pipeline Internals](pipeline.md#validation-retries-and-output)
|
||||
and the [binding reference](../config.md#module-bindings-and-validators).
|
||||
configured retry count reruns its complete stage attempt after an operational
|
||||
or structural error, module-requested retry, or actionable semantic rejection.
|
||||
An actionable module-requested retry and a validator rejection both use the
|
||||
same correction payload when the producer exposes an exact latest response;
|
||||
feedback-free module retries reconstruct the ordinary request without appended
|
||||
messages. The pipeline owns attempt lifecycle, validation chains, and retry
|
||||
diagnostics; see
|
||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||
[binding reference](../config.md#module-bindings-and-validators).
|
||||
|
||||
## Timeout Ownership
|
||||
|
||||
@@ -262,6 +283,13 @@ surfaced; when the completion already failed, its call error remains the
|
||||
result. Debug-bundle location, retention, and handling are operational concerns
|
||||
documented in [Operations](../operations.md#debug-bundles).
|
||||
|
||||
The attempt-terminal summary is a separate safe trace record: it contains
|
||||
attempt kinds, validator outcome counts and reason codes, effective policy,
|
||||
terminal action, and repair/usage references. It excludes raw assistant
|
||||
responses and correction text. Those values can appear only in the explicitly
|
||||
requested detailed prompt and response artifacts, which require sensitive-data
|
||||
handling.
|
||||
|
||||
Run manifests receive selected profile summaries, including optional effective
|
||||
backend and reasoning provenance, and component identities—not prompt, schema,
|
||||
source, reference, or response content. The published field semantics belong
|
||||
|
||||
@@ -22,6 +22,15 @@ to bindings whose declared execution class is `llm_backed` and rejects a
|
||||
binding-specific profile on a deterministic module. The user-facing precedence
|
||||
contract belongs in [Configuration](../config.md#pipelines).
|
||||
|
||||
An eligible LLM-backed chunk, extract, merge, or normalize producer may also
|
||||
declare correction protocol `single_response_v1`. That declaration is a
|
||||
promise that the implementation accepts one attempt-local semantic correction
|
||||
and returns an owned copy of the exact one model response that directly
|
||||
controlled the candidate. It must forward correction only to its structured
|
||||
completion request; it must not manufacture prior-response material by
|
||||
serializing a normalized artifact or expose opaque application IDs. Input,
|
||||
output, validator, and deterministic specs cannot declare the protocol.
|
||||
|
||||
Implementations that accept options must provide both an option validator and
|
||||
a builder. The validator is used while resolving configuration; the builder
|
||||
decodes the same options and constructs the implementation from the prepared
|
||||
@@ -36,6 +45,15 @@ they need, register each leaf implementation, and add any family-owned assets
|
||||
or default validator chains. They return contextual errors so production
|
||||
composition fails at startup rather than at the first run.
|
||||
|
||||
A validator that returns a completed rejection must supply two separate
|
||||
bounded values: a stable `ReasonCode` for provenance and actionable
|
||||
`CorrectionGuidance` for the producer. Guidance identifies the semantic defect
|
||||
and the constraints on one complete corrected replacement. It must not contain
|
||||
validator keys, diagnostic paths, opaque application IDs, or other internal
|
||||
identifiers. An operator-facing `Message` may explain the same event, but the
|
||||
framework never copies it into a model request. Missing or invalid guidance is
|
||||
a validator contract failure.
|
||||
|
||||
An artifact family can register an optional typed evidence projector alongside
|
||||
its codec. The projector returns defensive copies of the artifact's direct
|
||||
generic source references and must use the codec's exact Go type. It does not
|
||||
@@ -77,19 +95,40 @@ source-backed artifact-family normalizer projects its deterministic records
|
||||
into contextual candidates and owned typed record envelopes, supplies its
|
||||
chosen prompt identity and resolved LLM profile, and constructs an engine with
|
||||
explicit limits. The core filters invalid evidence, assigns contiguous
|
||||
request-local integer handles, renders bounded candidate and transcript
|
||||
materials, invokes the structured-completion boundary, and assesses the
|
||||
returned duplicate groups into a stable non-overlapping plan.
|
||||
one-based request-local candidate numbers, renders bounded candidate and
|
||||
transcript materials, invokes the structured-completion boundary, and assesses
|
||||
the returned duplicate groups into a stable non-overlapping plan. Candidate
|
||||
material declares the exact inclusive number range for the request. Transcript
|
||||
unit IDs and evidence range endpoints remain source positions in a separate
|
||||
namespace and are never valid candidate numbers.
|
||||
|
||||
The normalizer then applies that plan through a typed `ApplicationPolicy`. The
|
||||
core preserves ungrouped records, contribution order, and provenance while the
|
||||
artifact family owns group guards, field and evidence consolidation, durable
|
||||
ID derivation, retry and fallback presentation, warnings, and postconditions.
|
||||
Request-local handles do not enter the typed value or durable artifact. Fewer
|
||||
than two eligible candidates skips model invocation; exceeding a candidate or
|
||||
combined-material bound preserves the deterministic result under the family's
|
||||
fallback policy. Provider, transport, cancellation, and context-construction
|
||||
failures remain execution errors.
|
||||
ID derivation, retry and fallback presentation, classified diagnostics, and
|
||||
postconditions.
|
||||
These candidate numbers are the concrete private representation of ADR-0013's
|
||||
request-local handles; they do not enter the typed value or durable artifact.
|
||||
Fewer than two eligible candidates skips model invocation; exceeding a
|
||||
candidate or combined-material bound preserves the deterministic result under
|
||||
the family's fallback policy. Provider, transport, cancellation, and
|
||||
context-construction failures remain execution errors.
|
||||
|
||||
When the engine actually makes a proposal call, its typed result carries the
|
||||
owned exact proposal response under the same correction contract as other
|
||||
eligible producers. Deterministic skip, limit, and fallback outcomes carry no
|
||||
model candidate, so a later rejection applies terminal policy without spending
|
||||
an ineffective semantic retry.
|
||||
|
||||
When proposal assessment or typed group application rejects a structurally
|
||||
valid group, the normalizer may return its safe partial value with a
|
||||
module-requested retry. A feedback-capable directive supplies bounded
|
||||
model-facing correction guidance separately from operator diagnostics and
|
||||
retains the exact proposal response as its candidate. The shared stage retry
|
||||
mechanism appends that response and guidance to a fresh complete request. A
|
||||
structurally invalid completion has no valid candidate and therefore requests a
|
||||
feedback-free fresh attempt. On exhaustion, only the final safe fallback and
|
||||
its bounded process diagnostic advance to validation.
|
||||
|
||||
The core supplies a conservative generic prompt and the single private
|
||||
response schema. A domain prompt may substitute its semantic instructions but
|
||||
@@ -111,9 +150,13 @@ its domain prompt.
|
||||
3. Implement strict option decoding, construction, and the typed stage
|
||||
interface. Preserve caller ownership: do not retain mutable request data
|
||||
and return defensive copies where an implementation exposes stored data.
|
||||
If declaring correction capability, forward the request correction and
|
||||
retain only the exact validated response that controlled the result.
|
||||
4. Register the module through its typed registry helper and add it to the
|
||||
owning family registrar. Add a default validator chain only when that
|
||||
family owns the behavior; otherwise require an explicit compatible chain.
|
||||
Every rejection path in a validator must provide actionable correction
|
||||
guidance while retaining its stable internal reason code.
|
||||
5. Update the selectable-key and chain reference in
|
||||
[Configuration](../config.md#production-module-keys), the applicable
|
||||
integration contract, and focused tests. Keep the configuration document
|
||||
|
||||
@@ -12,7 +12,7 @@ own durable output shapes. Concrete production extensions are covered by
|
||||
The pipeline framework accepts a resolved composition, registries, shared
|
||||
dependencies, input bytes, a supplied prompt session, and state/debug
|
||||
collaborators. It returns logical output files, normalized artifacts, recorded
|
||||
rejections and warnings, manifest provenance, and checkpoint decisions. The
|
||||
rejections, grouped diagnostics, manifest provenance, and checkpoint decisions. The
|
||||
CLI owns process arguments, configuration discovery, session resolution,
|
||||
physical roots, and placement of returned output files.
|
||||
|
||||
@@ -32,8 +32,14 @@ Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
|
||||
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
||||
requested lanes where that is supported, resolves validator chains, checks
|
||||
module capabilities and typed artifact compatibility, validates options, and
|
||||
assigns a deterministic resolved-composition digest. The resolved pipeline
|
||||
contains bindings and declared reference targets, not external reference bytes.
|
||||
assigns a deterministic resolved-composition digest. A correction protocol is
|
||||
selected from each eligible LLM-backed producer specification and becomes part
|
||||
of that resolved identity; only `single_response_v1` is currently supported.
|
||||
Preparation rejects an LLM-backed producer that combines a non-empty validator
|
||||
chain with positive producer retries unless it declares that protocol. Producers
|
||||
without validators or without retries remain valid without correction support.
|
||||
The resolved pipeline contains bindings and declared reference targets, not
|
||||
external reference bytes.
|
||||
After selection, the resolver applies command, binding, and pipeline profile
|
||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||
an empty resolved binding profile. It resolves structural output repair
|
||||
@@ -59,9 +65,11 @@ remains declared but has no bytes until its producing step completes.
|
||||
|
||||
Preparation is the construction boundary. It validates the resolved shape and
|
||||
registry set, clones the resolved data, then constructs the input adapter,
|
||||
chunker, stage-local validators, every typed lane, and output encoder. Each
|
||||
registered builder receives its own cloned build request immediately before its
|
||||
module-owned code runs. Preparation also collects stable checkpoint
|
||||
chunker, stage-local validators, every typed lane, and output encoder. The
|
||||
prepared producer metadata preserves each selected correction protocol, and
|
||||
the resolved digest carrying that metadata participates in checkpoint identity.
|
||||
Each registered builder receives its own cloned build request immediately
|
||||
before its module-owned code runs. Preparation also collects stable checkpoint
|
||||
fingerprints. Missing registrations, incompatible typed entries, nil
|
||||
implementations, and constructor failures are reported before source parsing
|
||||
or any stage operation begins.
|
||||
@@ -89,6 +97,13 @@ codec, checks its complete schema and media identity, and records a content
|
||||
digest plus bounded producer provenance. A missing, ambiguous, invalid, or
|
||||
incompatible producer prevents the consumer step from starting.
|
||||
|
||||
Resolution receives only exact stage-and-lane operational reference overrides.
|
||||
The CLI may offer broader pipeline- or lane-scoped selectors, but expands and
|
||||
arbitrates those before entering the framework. External overrides are applied
|
||||
after configured external defaults and local bindings. They cannot coexist
|
||||
with a generated binding for the same target and slot, and external unbinds do
|
||||
not remove generated handoffs.
|
||||
|
||||
## Execution And Ordering
|
||||
|
||||
The runner validates its input, installs no-op state collaborators when none
|
||||
@@ -128,17 +143,55 @@ their target: chunks, codec-decoded typed candidates, or serialized codec
|
||||
bytes. Each typed validator receives a newly decoded value from the one
|
||||
candidate serialization for that attempt, while serialized validators receive
|
||||
separately owned representation bytes and schema metadata. They may approve,
|
||||
approve with warnings, reject, or fail. A rejection is an ordinary pipeline
|
||||
result; a validator error is a framework error.
|
||||
approve with warnings, reject, fail, or be skipped when a runtime prerequisite
|
||||
is unavailable. The shared executor settles every configured validator in
|
||||
order. A failed LLM-backed validator retries only itself against the same
|
||||
immutable candidate; it does not regenerate the producer or alter the
|
||||
validator request. Rejections stop that validator, while other configured
|
||||
validators still run. The executor retains ordered results, bounded
|
||||
deduplicated correction guidance from rejections, and only the final exhausted
|
||||
failure outcome for each validator. The correction builder keeps first
|
||||
occurrence order, omits internal reason codes, validator names, and operator
|
||||
messages, and requests one complete replacement. Missing guidance or an
|
||||
oversized aggregate is a framework contract error; guidance is never inferred
|
||||
or truncated.
|
||||
|
||||
The runner applies the binding's retry policy around a stage operation and its
|
||||
complete validation chain. It preserves warnings only from the final accepted
|
||||
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
|
||||
directives consume this same budget and validate any final safe fallback through
|
||||
the normalizer chain.
|
||||
complete validation chain. It preserves terminal diagnostics only from the final accepted
|
||||
or rejected attempt, plus one fixed validation-incomplete warning per validator whose execution
|
||||
budget was exhausted under `warn_continue`. Cancellation stops retries.
|
||||
Normalizer-specific retry directives consume this same budget and validate any
|
||||
final safe fallback through the normalizer chain. A directive may carry bounded
|
||||
correction guidance only when it also exposes the exact latest
|
||||
`single_response_v1` candidate. The state machine then uses the same replacement
|
||||
request shape as validator correction. A directive without guidance clears any
|
||||
prior correction and starts a fresh attempt, which preserves structural retry
|
||||
behavior when no valid response exists.
|
||||
|
||||
The artifact-neutral producer-attempt state machine owns that shared budget,
|
||||
attempt provenance, semantic-correction material, and terminal-policy
|
||||
selection. It accepts producer and complete-validation closures, so artifact
|
||||
materialization, cache handling, checkpoints, and debug output stay at the
|
||||
operation boundary. It distinguishes operational, structural, module-requested,
|
||||
and validator-semantic retries. Model feedback from either semantic source is
|
||||
available only for a valid latest `single_response_v1` candidate. A
|
||||
deterministic or no-model validator rejection instead settles the semantic
|
||||
policy immediately, while a feedback-free module directive remains an ordinary
|
||||
fresh retry. Structural-output errors alone use the structural policy, and
|
||||
validation failure without rejection settles the validator-failure policy
|
||||
without regenerating the producer.
|
||||
|
||||
Chunk planning uses this state machine for generated plans. A rejected or
|
||||
validation-incomplete automatic cache hit is not model material and therefore
|
||||
falls through to a fresh initial generation at producer attempt one; it neither
|
||||
receives a correction, consumes retry budget, promotes cached-candidate
|
||||
diagnostics, nor overwrites the stored record. An incomplete cache validation
|
||||
under `fail_run` terminates instead. Only a newly generated, completely
|
||||
validated plan is published to the chunk-plan store. Rejected plans never
|
||||
advance, and validation-incomplete plans remain unpublishable.
|
||||
|
||||
After terminal lane work, the runner assembles manifest provenance, normalized
|
||||
artifacts, rejections, warnings, and an optional accepted chunk map. When an
|
||||
artifacts, rejections, final grouped diagnostics, and an optional accepted chunk map. When an
|
||||
output policy selected evidence lanes, it decodes accepted serialized normalize
|
||||
outputs through their registered codecs and invokes the prepared typed
|
||||
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
|
||||
@@ -149,6 +202,15 @@ The CLI publishes those files only after the runner returns without a framework
|
||||
error. Logical file names and schemas are defined by the [output integration
|
||||
contracts](../integrations/).
|
||||
|
||||
For every completed producer disposition, the runner projects one bounded
|
||||
validation summary to the manifest, the affected rejection when present, and
|
||||
the CLI result receipt. The summary records status, configured-order rejecting
|
||||
validators and reason codes, incomplete validators, producer-attempt count,
|
||||
and terminal action. It contains no operator message, correction guidance, or
|
||||
model response. `complete`, `rejected`, and `incomplete` describe the final
|
||||
candidate disposition; a run-level `incomplete` status indicates at least one
|
||||
current-run output advanced under `warn_continue`.
|
||||
|
||||
## Checkpoint And Debug Hooks
|
||||
|
||||
The runner receives checkpoint and debug interfaces rather than roots. It
|
||||
@@ -158,10 +220,26 @@ handoff. Generated-reference dependencies participate in checkpoint decisions.
|
||||
Selective recomputation can require a canonical accepted normalized predecessor
|
||||
before a dependent lane starts.
|
||||
|
||||
The runner writes successful checkpoint artifacts only after complete accepted
|
||||
validation. Chunk plans follow the same rule for publication. A rejection,
|
||||
invalid structured response, or incomplete validation is never reusable state;
|
||||
the current run may still hand off an otherwise valid `warn_continue` result
|
||||
according to its terminal policy. The runner carries private reuse eligibility
|
||||
through extract, merge, normalize, and generated-reference handoff. Any stage
|
||||
derived from incomplete validation skips both checkpoint lookup and all
|
||||
checkpoint publication even when that stage's own validation completes.
|
||||
External references and fully validated generated references remain eligible.
|
||||
Attempt debug records retain safe kind,
|
||||
validator, repair-usage, policy, and terminal-decision provenance. Full
|
||||
assistant and correction content remains confined to the requested detailed
|
||||
LLM trace.
|
||||
|
||||
Debug recording is attempt-scoped and application-owned. A failure to persist
|
||||
required debug data is a framework error. State roots, persistence, reason-code
|
||||
meanings, resume, and cleanup are intentionally owned by
|
||||
[Run State Internals](state.md) and [Operations](../operations.md).
|
||||
Extract-validator trace scopes include the current chunk ordinal so concurrent
|
||||
chunks cannot overwrite one another's validator attempts or LLM artifacts.
|
||||
|
||||
## Invariants To Preserve
|
||||
|
||||
|
||||
@@ -64,12 +64,13 @@ Ordinary resume loads extract, merge, and normalize checkpoints progressively
|
||||
and may execute later lane stages after an earlier cache miss. Selective
|
||||
recomputation instead asks the loader for the required producer's accepted
|
||||
normalize artifact. That lookup reuses the existing normalize files, requires
|
||||
workspace schema v3 plus an exact non-empty invocation identity, and deliberately
|
||||
workspace schema v4 plus an exact non-empty invocation identity, and deliberately
|
||||
does not require extract or merge checkpoint files or dependency fingerprints.
|
||||
The runner performs canonical codec and producer-provenance validation before
|
||||
cloning the artifact into normal step output. Success restores only stored
|
||||
normalize warnings and emits one normalize decision; failure retains the files,
|
||||
records the decision, and stops without executing the producer or consumer.
|
||||
normalize diagnostics and emits one normalize decision; failure retains the
|
||||
files, records the decision, and stops without executing the producer or
|
||||
consumer.
|
||||
|
||||
The loader assigns a typed category and reason code at each validation site;
|
||||
diagnostic prose is not classified after the fact. The runner then applies
|
||||
@@ -97,7 +98,7 @@ owns the operator workflow and stable reason-code meanings.
|
||||
|
||||
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
|
||||
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
|
||||
resolution, run, warning, and failure artifacts. `internal/framework/debug`
|
||||
resolution, run, final grouped diagnostic, and failure artifacts. `internal/framework/debug`
|
||||
implements the pipeline-facing trace recorder under the trace root.
|
||||
|
||||
The CLI allocates a bundle before pipeline resolution and treats requested
|
||||
|
||||
@@ -105,10 +105,40 @@ and resolves configuration before module preparation and source parsing. It
|
||||
then performs any permitted cache lookup, executes the pipeline, and publishes
|
||||
logical output files only after a successful runner result.
|
||||
|
||||
On success, the command reports the output bundle path. A warning-bearing run
|
||||
still succeeds and reports its warning count on standard error. Errors and
|
||||
On success, the command reports the output bundle path. A run with actionable
|
||||
process warnings still succeeds and reports warning-group and occurrence counts
|
||||
on standard error; advisory and observation findings do not produce a warning
|
||||
line. Errors and
|
||||
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
||||
|
||||
## Validation Retries And Terminal Outcomes
|
||||
|
||||
Each producer binding has one outer **retries** budget. It covers complete
|
||||
producer attempts for operational failures, invalid structured output,
|
||||
normalizer fallback retry, and semantic correction. It is independent from
|
||||
PromptKit's structural-repair calls inside one completion and from an
|
||||
LLM-backed validator's own retry budget. A semantic correction rebuilds the
|
||||
ordinary producer request and supplies only the latest rejected model response
|
||||
plus aggregated validator guidance; it is not a conversation replay.
|
||||
|
||||
After the applicable budgets are exhausted, the resolved
|
||||
[`validation_policy`](config.md#pipelines) determines the result. Structural
|
||||
failure and semantic rejection normally fail the run; an explicit
|
||||
`reject_output` records a rejection and allows unrelated work to finish. A
|
||||
validator execution failure normally uses `warn_continue`, which keeps an
|
||||
otherwise accepted result in the current run with `incomplete` validation
|
||||
provenance. It emits one bounded warning for every validator whose execution
|
||||
budget was exhausted. A corrected result that later passes validation does not
|
||||
retain diagnostics from abandoned attempts.
|
||||
|
||||
Treat a successful process exit as a completed run, not as proof that every
|
||||
candidate was fully validated. Inspect the receipt's `validation_status`,
|
||||
`validation_summaries`, rejection count, and warning group and occurrence
|
||||
counts when an orchestrator requires complete validation. The durable fields
|
||||
and their meanings are owned by the
|
||||
[run-result receipt](integrations/run-result.md) and
|
||||
[published JSON output contract](integrations/json-output.md).
|
||||
|
||||
## Output Bundles
|
||||
|
||||
Each successful run receives a generated safe run identifier and writes beneath:
|
||||
@@ -158,7 +188,9 @@ The configured cache mode controls one invocation:
|
||||
A reused plan is still materialized and validated against the current source.
|
||||
If a prior plan no longer gives acceptable results, use a refresh run rather
|
||||
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
||||
chunking work.
|
||||
chunking work. A plan accepted only under incomplete validation is not
|
||||
published, and a rejected cache hit falls through to ordinary generation rather
|
||||
than becoming a correction candidate.
|
||||
|
||||
## Checkpoint Recording, Resume, And Recompute
|
||||
|
||||
@@ -183,6 +215,12 @@ Reasoning-effort inheritance, replacement, and explicit clearing are distinct
|
||||
runtime identities, so checkpoints created under one state are not reused by
|
||||
either of the others.
|
||||
|
||||
Only accepted, completely validated chunk, extract, merge, and normalize
|
||||
results are checkpointed for reuse. Rejected, structurally invalid, and
|
||||
validation-incomplete producer results remain non-reusable, even when a
|
||||
`warn_continue` result advanced during its original run. A resumed invocation
|
||||
therefore reruns that producer rather than treating degraded state as accepted.
|
||||
|
||||
Checkpoint state is confined below an identity-specific path:
|
||||
|
||||
~~~
|
||||
@@ -237,13 +275,16 @@ Only a [debug-enabled run](cli.md#run) creates a bundle:
|
||||
~~~
|
||||
|
||||
The summary contains redacted invocation and resolution information plus run,
|
||||
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
|
||||
contains allowlisted application diagnostic records and can include source or
|
||||
derived application data. Neither surface is a cache input. Do not treat a
|
||||
debug bundle as safe to share merely because its configuration summary is
|
||||
redacted. Invocation metadata omits reasoning effort when it is inherited,
|
||||
records the replacement value when one is supplied, and records an empty value
|
||||
when inherited reasoning was explicitly cleared.
|
||||
final grouped diagnostic, checkpoint, chunk-plan, and terminal reporting artifacts. Attempt
|
||||
terminal records contain bounded attempt kinds, validator outcomes, policy,
|
||||
decision, PromptKit repair count, and usage; they do not contain assistant
|
||||
responses or complete correction messages. The trace contains allowlisted
|
||||
application diagnostic records and can include source, model, and correction
|
||||
content. Neither surface is a cache input. Do not treat a debug bundle as safe
|
||||
to share merely because its configuration summary is redacted. Invocation
|
||||
metadata omits reasoning effort when it is inherited, records the replacement
|
||||
value when one is supplied, and records an empty value when inherited reasoning
|
||||
was explicitly cleared.
|
||||
|
||||
Notarius never creates debug state without an explicit request and never
|
||||
automatically deletes a requested bundle. If allocation succeeds, the command
|
||||
|
||||
@@ -146,8 +146,9 @@ lanes, validators, and LLM profile: the canonical source digest selects the
|
||||
plan, while the current run still applies its configured chunk validators to
|
||||
the materialized chunks.
|
||||
|
||||
The framework owns orchestration and handoff provenance. Modules return logical
|
||||
results and warnings; they do not own CLI reporting, physical output, cache, or
|
||||
The framework owns orchestration, origin enrichment, aggregation, and handoff
|
||||
provenance. Modules return logical results and classified diagnostics; they do
|
||||
not own CLI reporting, physical output, cache, or
|
||||
debug roots, durable file placement, or checkpoint and debug lifecycle.
|
||||
|
||||
After pipeline-wide chunking, extraction uses bounded framework concurrency.
|
||||
@@ -159,26 +160,53 @@ may overlap. The framework must not create unbounded goroutines per lane or
|
||||
chunk.
|
||||
|
||||
Completion timing does not choose public ordering or errors. The coordinator
|
||||
orders accepted artifacts, warnings, rejections, checkpoint events, and
|
||||
orders accepted artifacts, grouped diagnostics, rejections, checkpoint events, and
|
||||
framework errors by stable pipeline scope. Rejections do not cancel unrelated
|
||||
work. A framework error cancels derived work, prevents undispatched work from
|
||||
starting, waits for started work, and prevents output encoding.
|
||||
|
||||
Warnings are process-only signals: configuration degradation, approved fallback,
|
||||
or incomplete configured validation. Quality uncertainty and grounding findings
|
||||
are advisories; successful canonicalization and cleanup are observations.
|
||||
Modules choose that semantic classification, while the framework attaches
|
||||
origin, aggregates groups, enforces bounds, and presents final collections.
|
||||
An ordinary successful run therefore has zero warnings. See
|
||||
[ADR-0015](../adr/0015-separate-process-warnings-from-quality-diagnostics.md)
|
||||
for the decision rationale.
|
||||
|
||||
## Validation
|
||||
|
||||
Validation is a framework-managed boundary around outputs from chunk, extract,
|
||||
merge, and normalize stages. Validators receive immutable stage output
|
||||
and make an explicit whole-output decision: approve, approve with warnings, or
|
||||
reject.
|
||||
merge, and normalize stages. Validators receive immutable stage output and
|
||||
make an explicit whole-output decision: approve, reject, fail, or skip when a
|
||||
runtime prerequisite is unavailable.
|
||||
|
||||
Typed artifact validators receive the domain value directly. Chunk validators
|
||||
receive source-zone chunks, while serialized validators receive immutable
|
||||
representation bytes and declared schema metadata. A validator registered for
|
||||
one target or artifact kind cannot satisfy an incompatible selection.
|
||||
|
||||
Rejection is a recorded pipeline outcome, not a framework execution error.
|
||||
Validator execution failures are framework errors. Rejected output does not
|
||||
advance to the next stage.
|
||||
The framework runs every applicable validator sequentially in configured order.
|
||||
It aggregates rejections, exhausted validator failures, and skips before the
|
||||
producer policy chooses a disposition. A completed rejection never advances.
|
||||
With no rejection, an exhausted validator failure may fail the run or, under
|
||||
the configured `warn_continue` policy, advance a structurally valid candidate
|
||||
with explicit incomplete-validation provenance. Validators report findings;
|
||||
they do not choose candidate disposition.
|
||||
|
||||
A completed rejection supplies a stable reason code for internal provenance
|
||||
and bounded actionable correction guidance for the candidate producer. Reason
|
||||
codes, validator keys, and operator-facing messages remain diagnostic data;
|
||||
they are not model instructions. The framework constructs model-facing retry
|
||||
text only from the semantic guidance and fails the contract rather than
|
||||
inventing or truncating missing guidance.
|
||||
|
||||
An LLM-backed module may also request a feedback-aware retry when its own
|
||||
deterministic translation or typed safety policy rejects a structurally valid
|
||||
model response. It must supply model-facing guidance separately from its
|
||||
reason code, operator message, and fallback diagnostics, together with the
|
||||
exact `single_response_v1` candidate. A feedback-free module retry remains
|
||||
valid when no exact candidate exists.
|
||||
|
||||
Default validator chains are production composition policy and are registered
|
||||
centrally by stage and module. Configuration may replace a stage-local default,
|
||||
@@ -199,6 +227,17 @@ PromptKit owns bounded structural correction within one structured completion.
|
||||
Notarius owns outer stage attempts, semantic validation, and acceptance policy;
|
||||
the two budgets must remain separate.
|
||||
|
||||
An LLM-backed producer can participate in semantic correction only when it
|
||||
declares `single_response_v1` and returns the exact one response that directly
|
||||
controlled its candidate. On an actionable validator rejection or
|
||||
feedback-capable module retry, the framework rebuilds the ordinary request and
|
||||
appends only the latest defective response as an `assistant` message plus one
|
||||
aggregated `user` correction message. This is a fresh replacement request, not
|
||||
a growing conversation. The retry budgets, terminal policy, and sensitive-data
|
||||
rationale are recorded in
|
||||
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md) and
|
||||
[ADR-0016](../adr/0016-feedback-aware-module-requested-retries.md).
|
||||
|
||||
When a model selects an application entity, callers must supply a contextual
|
||||
selection and deterministically attach the opaque application identity whenever
|
||||
the selection resolves exactly. Models do not receive or reproduce opaque
|
||||
@@ -233,7 +272,8 @@ invalid or incompatible.
|
||||
|
||||
Run manifests record enough resolved pipeline, module, source, reference, and
|
||||
LLM provenance to make a run auditable after configuration changes. Manifests
|
||||
record identities and summaries rather than secret or large payload content.
|
||||
record identities and bounded validation summaries rather than secret, raw
|
||||
model, correction, or large payload content.
|
||||
|
||||
## State, Output, And Safety
|
||||
|
||||
@@ -253,12 +293,23 @@ an invocation that explicitly requests resume. Debug is never a cache input and
|
||||
is never created without an explicit request. Pipeline modules receive
|
||||
collaborator interfaces and never physical roots.
|
||||
|
||||
Only accepted, completely validated producer output is reusable checkpoint or
|
||||
chunk-plan state. Rejected, structurally invalid, and validation-incomplete
|
||||
results cannot become cache or checkpoint inputs, even when a
|
||||
`warn_continue` result is allowed to advance in the current run. This
|
||||
ineligibility follows derived merge and normalize results and generated
|
||||
references for the remainder of the run: current-run handoff remains allowed,
|
||||
but no dependent cache or checkpoint may be loaded or published.
|
||||
|
||||
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
||||
deletion must be narrow and explicit. Notarius never automatically deletes
|
||||
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
||||
|
||||
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
||||
traces, manifests, documentation, examples, or redacted configuration. Debug
|
||||
traces, manifests, documentation, examples, or redacted configuration. Raw
|
||||
assistant responses and complete correction messages are attempt-local and are
|
||||
excluded from ordinary durable records and summaries; the requested detailed
|
||||
debug trace is the sole diagnostic surface allowed to retain them. Debug
|
||||
collection is allowlisted to application-owned payloads and must not capture
|
||||
unrelated process environment values or filesystem content. Trace data may
|
||||
contain application data and therefore inherits its sensitivity; operators own
|
||||
|
||||
83
docs/releases/v0.4.0.md
Normal file
83
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Notarius v0.4.0
|
||||
|
||||
This release strengthens LLM reliability and validation throughout the
|
||||
configured pipeline, upgrades the PromptKit integration, and establishes the
|
||||
source-release and downstream-consumer workflows needed for broader D&D
|
||||
pipeline integration.
|
||||
|
||||
## Summary
|
||||
|
||||
Notarius now distinguishes PromptKit structural-output repair from
|
||||
application-owned semantic validation retries. Producer candidates can run
|
||||
through complete deterministic validator chains, receive bounded semantic
|
||||
correction guidance, and retry under explicit stage policies. Final run
|
||||
receipts and manifests preserve bounded validation provenance, while outputs
|
||||
that advance with incomplete validation remain available to the current run
|
||||
without entering reusable checkpoint state.
|
||||
|
||||
The release also adds a maintained complete D&D subprocess-consumer workflow,
|
||||
diagnostic build versions, and the source-only release procedure used to
|
||||
publish this version.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- Configuration files must use schema version 4. Version 3 is not decoded or
|
||||
rewritten; rename the top-level `scriptorium` section to `promptkit` when
|
||||
migrating. See [Configuration](../config.md#migrating-version-3-configuration).
|
||||
- PromptKit is pinned to v0.9.0. Operator profile files use PromptKit's v0.9.0
|
||||
format and may use its profile-inheritance support. Notarius continues to
|
||||
resolve operator profiles before embedded fallbacks.
|
||||
- Structural-output repair and semantic stage retries are separate bounded
|
||||
mechanisms. Maintained production prompts request one structural repair by
|
||||
default; explicit configuration can override the supported repair count.
|
||||
- Validation policy can now fail a run, reject an output, or permit an
|
||||
otherwise valid candidate to advance with incomplete-validation provenance.
|
||||
The application defaults are documented in
|
||||
[Configuration](../config.md#pipelines).
|
||||
- The `notarius.run-result.v1` receipt remains at schema version 1 and adds
|
||||
optional validation summaries plus a required validation-status field.
|
||||
Consumers of this pre-release contract should follow the current
|
||||
[run-result receipt](../integrations/run-result.md).
|
||||
- Existing D&D artifact schema identities remain unchanged. Validation and
|
||||
producer-policy changes can nevertheless cause previously accepted weak
|
||||
candidates to retry, reject, or fail instead.
|
||||
|
||||
## Upgrade
|
||||
|
||||
1. Migrate every Notarius configuration to version 4 and rename `scriptorium`
|
||||
to `promptkit`.
|
||||
2. Review deployed PromptKit profiles against the pinned v0.9.0 profile format
|
||||
and ensure their credential environment variables are available at run
|
||||
time.
|
||||
3. Run `notarius config validate --config <path> --pipeline <id>` before the
|
||||
first production invocation.
|
||||
4. Review `structured_output_repair_attempts`, producer retry counts, and
|
||||
`validation_policy` wherever the deployment needs behavior different from
|
||||
the documented defaults.
|
||||
5. Update subprocess consumers to inspect receipt `validation_status` and to
|
||||
tolerate the optional bounded `validation_summaries` field. A consumer that
|
||||
requires fully validated artifacts should require `approved`.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded PromptKit from v0.5.0 through v0.9.0 and adopted profile
|
||||
inheritance, structured-output repair, typed error classification, and the
|
||||
correction-aware completion protocol.
|
||||
- Added pipeline and binding configuration for structural repair and terminal
|
||||
validation policy, with strict startup validation and effective-setting
|
||||
provenance.
|
||||
- Added feedback-aware retries for chunking, extraction, merge, normalize, and
|
||||
semantic reconciliation producers. Retry prompts contain the exact defective
|
||||
response and actionable semantic correction guidance without exposing
|
||||
internal reason codes or opaque entity identifiers.
|
||||
- Added complete validator-chain execution, validator retry handling, bounded
|
||||
warnings, terminal dispositions, and durable validation summaries.
|
||||
- Prevented validation-incomplete artifacts and all derived lineage from
|
||||
loading or publishing reusable checkpoints while preserving same-run
|
||||
generated-reference handoff.
|
||||
- Tightened cached chunk-plan validation so only completely validated plans are
|
||||
reused or replace stored plans.
|
||||
- Added a machine-readable subprocess receipt workflow and complete D&D
|
||||
consumer documentation covering all maintained artifacts.
|
||||
- Added source-release checks, immutable lightweight-tag guidance, Linux and
|
||||
Darwin build verification, and diagnostic `notarius --version` output.
|
||||
85
docs/releases/v0.5.0.md
Normal file
85
docs/releases/v0.5.0.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# Notarius v0.5.0
|
||||
|
||||
This release separates actionable process warnings from extraction-quality
|
||||
advisories and routine normalization observations, giving operators a quiet
|
||||
warning channel without discarding durable diagnostic detail.
|
||||
|
||||
## Summary
|
||||
|
||||
Notarius now carries one validated, origin-aware diagnostic contract from
|
||||
producers and validators through retries, reusable state, output publication,
|
||||
debug summaries, run receipts, and CLI presentation. Warnings are reserved for
|
||||
process degradation or incomplete configured work. Data-quality findings are
|
||||
advisories, and successful deterministic cleanup is recorded as observations.
|
||||
An ordinary successful run therefore reports zero warnings while retaining
|
||||
bounded diagnostic provenance for later review.
|
||||
|
||||
The framework aggregates findings deterministically by their stable identity
|
||||
and complete pipeline origin, preserves exact occurrence counts, and retains
|
||||
bounded representative samples. Warning groups fail rather than truncate;
|
||||
advisory and observation representation is bounded with explicit truncation
|
||||
metadata and exact unrepresented-occurrence counts.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- `warnings.json` now uses the incompatible grouped
|
||||
`notarius.warnings.v2` envelope and contains process warnings only. Consumers
|
||||
of the former flat warning payload must migrate to the current
|
||||
[JSON output contract](../integrations/json-output.md).
|
||||
- The new `diagnostics.json` file uses `notarius.diagnostics.v1` and contains
|
||||
advisory and observation groups. Production `index.json` files always expose
|
||||
both `warnings_file` and `diagnostics_file`.
|
||||
- The machine-readable run receipt is now `notarius.run-result.v2`. It replaces
|
||||
`warning_count` with exact warning group and occurrence counts and adds
|
||||
advisory/observation group, occurrence, and truncation fields. See the
|
||||
current [run-result receipt](../integrations/run-result.md).
|
||||
- Custom output modules must return their complete logical file set or an
|
||||
error. The former `OutputResult.Warnings` field has been removed; an output
|
||||
module cannot report a warning after serializing its output.
|
||||
- Reusable state now uses `notarius.workspace.v4` and chunk-plan records use
|
||||
`notarius.chunk-plan.v3` so they can preserve structured diagnostics. Older
|
||||
pre-release reusable state is not reused under these contracts; start with
|
||||
clean state when deterministic continuity with an older workspace is not
|
||||
required.
|
||||
- Validation acceptance, semantic retry budgets, rejection policy, and D&D
|
||||
artifact schema identities are unchanged by this release.
|
||||
|
||||
## Upgrade
|
||||
|
||||
1. Update subprocess consumers to require `notarius.run-result.v2` and read
|
||||
`warning_group_count`, `warning_occurrence_count`,
|
||||
`diagnostic_group_count`, `diagnostic_occurrence_count`, and
|
||||
`diagnostics_truncated`.
|
||||
2. Update output-bundle consumers to decode `notarius.warnings.v2`, discover
|
||||
`diagnostics.json` through `index.json`, and treat diagnostics as review
|
||||
information rather than process warnings.
|
||||
3. Update any custom output module for the removal of
|
||||
`OutputResult.Warnings`; return an error when encoding cannot complete.
|
||||
4. Clear pre-release reusable state before the first upgraded production run
|
||||
when deterministic continuity with an older workspace is not required.
|
||||
5. Run `notarius config validate --config <path> --pipeline <id>` and perform
|
||||
one representative run before promoting the release in an automated
|
||||
pipeline.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added validated diagnostic dispositions, categories, origins, stable reason
|
||||
codes, exact occurrence counts, and bounded representative samples.
|
||||
- Added deterministic run-level aggregation with separate limits for
|
||||
actionable warning groups and advisory/observation groups.
|
||||
- Reclassified D&D source-relatedness and unresolved-identity findings as
|
||||
data-quality advisories and routine normalization changes as observations.
|
||||
- Preserved structured diagnostics across producer retries, validation,
|
||||
generated-reference handoff, checkpoints, chunk-plan reuse, and debug
|
||||
summaries while discarding superseded-attempt findings.
|
||||
- Added grouped `warnings.json`, a new grouped `diagnostics.json`, and the
|
||||
corresponding production index entries.
|
||||
- Upgraded the machine-readable run receipt and human CLI summary to report
|
||||
exact warning and diagnostic counts without allowing advisory volume to
|
||||
create warning output.
|
||||
- Removed post-encoding output warnings and hardened diagnostic validation,
|
||||
overflow handling, aggregate memory bounds, and warning-file path
|
||||
presentation.
|
||||
- Documented diagnostic ownership, classification, operator interpretation,
|
||||
durable contracts, and architectural invariants in ADR-0015 and the
|
||||
canonical CLI, operations, integration, and internal documentation.
|
||||
84
docs/releases/v0.6.0.md
Normal file
84
docs/releases/v0.6.0.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Notarius v0.6.0
|
||||
|
||||
This release improves the reliability and ergonomics of unattended,
|
||||
subprocess-driven D&D extraction pipelines.
|
||||
|
||||
## Summary
|
||||
|
||||
Notarius now gives models more precise, semantically useful correction guidance
|
||||
when deterministic or model-backed validation rejects an otherwise structured
|
||||
candidate. Semantic reconciliation retries identify candidates with compact,
|
||||
request-local numbers, preserve valid candidates when a proposal cannot be
|
||||
repaired, and report bounded process warnings when reconciliation is
|
||||
incomplete. D&D extraction also canonicalizes safely resolvable reversed source
|
||||
ranges and applies more consistent schema, prompt, and validator constraints,
|
||||
improving successful operation with smaller models without weakening evidence
|
||||
validation.
|
||||
|
||||
An optional LLM-backed combat-semantics validator can review whether scene
|
||||
descriptions correctly consolidate combat and apply the `combat` kind. For
|
||||
subprocess callers, command-line references now follow the pipeline-scoped
|
||||
configuration model: a shared reference can be supplied once and automatically
|
||||
reach every compatible selected target, while lane- and binding-specific forms
|
||||
remain available for exceptions.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- The meaning of an unqualified `--reference slot=path` or
|
||||
`--without-reference slot` selector has changed. It now applies to every
|
||||
selected pipeline target that declares the slot instead of requiring exactly
|
||||
one matching target. Review callers that relied on ambiguity rejection or a
|
||||
unique implicit target; the current selector contract is documented in the
|
||||
[CLI reference](../cli.md).
|
||||
- The stage-wide `merge.slot` CLI reference shorthand has been removed. Use
|
||||
`lane.merge.slot` for an exact merge binding, `lane.slot` for all compatible
|
||||
bindings in one lane, or an unqualified `slot` for pipeline scope.
|
||||
- Command-line file references cannot replace, remove, or coexist with a
|
||||
generated artifact handoff for the same concrete target and slot. Resolution
|
||||
reports the conflict so the caller can narrow or remove the selector.
|
||||
- The private LLM-facing semantic-reconciliation contract now uses contiguous
|
||||
`candidate_number` values rather than application identities. This is not a
|
||||
durable or operator-configurable contract and requires no operator action.
|
||||
- Durable D&D artifact schemas, the `notarius.run-result.v2` receipt,
|
||||
`notarius.warnings.v2`, and `notarius.diagnostics.v1` are unchanged from
|
||||
`v0.5.0`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
1. Update subprocess commands to provide shared inputs once with
|
||||
`--reference slot=path`, and convert any stage-wide `merge.slot` selector to
|
||||
an explicit supported scope. The [subprocess consumer guide](../consumers/subprocess.md)
|
||||
and [complete D&D consumer guide](../consumers/dnd-pipeline.md) show the
|
||||
current invocation pattern.
|
||||
2. Review CLI reference overrides that overlap generated same-run references;
|
||||
narrow or remove an external selector rather than attempting to replace the
|
||||
generated handoff.
|
||||
3. Run `notarius config validate --config <path> --pipeline <id>` for every
|
||||
deployed pipeline configuration.
|
||||
4. Optionally add
|
||||
`extract/dnd/scene-descriptions/combat_semantics` to the scene-description
|
||||
validator chain where the deployment wants LLM-backed combat-scene review;
|
||||
see the [configuration reference](../config.md).
|
||||
5. Perform a representative run with the deployed model profile and inspect
|
||||
the machine-readable receipt, warnings, and diagnostics before promotion.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added feedback-aware, module-requested retry support for normalizers while
|
||||
keeping retry budgets bounded and preserving attempt diagnostics.
|
||||
- Reworked semantic reconciliation around compact, contiguous candidate
|
||||
numbers, explicit namespace guidance, complete proposal validation, and
|
||||
model-facing retry feedback that omits opaque internal identities.
|
||||
- Preserved candidates separately when an exhausted semantic proposal cannot
|
||||
be safely applied, with a bounded process warning describing the fallback.
|
||||
- Canonicalized safely resolvable reversed D&D source-reference endpoints
|
||||
before deterministic coverage validation and clarified the shared evidence
|
||||
prompt.
|
||||
- Improved D&D schemas, prompts, and validators with closed-value constraints,
|
||||
contextual correction guidance, shared source-range coverage logic, and
|
||||
clearer item-holder transition rules.
|
||||
- Added the optional D&D scene-description combat-semantics validator, shared
|
||||
combat policy assets, evaluation fixtures, and retry-path coverage.
|
||||
- Made unqualified CLI references pipeline-scoped, added hierarchical lane and
|
||||
exact-binding selectors, defined deterministic override precedence, and
|
||||
documented first-class subprocess use.
|
||||
648
docs/roadmap/archive/warning-signal-and-presentation-audit.md
Normal file
648
docs/roadmap/archive/warning-signal-and-presentation-audit.md
Normal file
@@ -0,0 +1,648 @@
|
||||
# Warning Signal And Presentation Audit
|
||||
|
||||
## Executive Assessment
|
||||
|
||||
Notarius warning execution is mechanically stronger than its operator-facing
|
||||
presentation. Terminal-attempt promotion, stable ordering after concurrent
|
||||
work, checkpoint replay, validation summaries, and debug retention are all
|
||||
substantially correct. The audit found no general duplicate-append defect in
|
||||
the extract, merge, or normalize handoffs and no leakage of abandoned-attempt
|
||||
warnings into a successful result.
|
||||
|
||||
The warning channel itself is not coherent. One flat `contracts.Warning` type
|
||||
currently represents at least four materially different concepts:
|
||||
|
||||
- actionable degradation or incomplete validation;
|
||||
- heuristic data-quality doubt;
|
||||
- successful but potentially reviewable fallback; and
|
||||
- routine canonicalization, ordering, and deduplication observations.
|
||||
|
||||
That conflation is the primary reason successful runs produce a count that is
|
||||
large but operationally weak. The maintained complete D&D example demonstrates
|
||||
the problem without a live provider: an approved run with no rejected outputs
|
||||
published 12 warning records, all from three advisory relatedness checks. An
|
||||
operator separately reported a successful complete D&D run with 10 outputs,
|
||||
one rejection, and 85 warnings. The production bundle for that run was not
|
||||
available in this environment, so its reason-code distribution could not be
|
||||
measured.
|
||||
|
||||
The current implementation also has four correctness or robustness gaps:
|
||||
|
||||
1. warning records lose stage, step, lane, module, validator, and chunk
|
||||
provenance when promoted, which makes safe aggregation and diagnosis
|
||||
impossible from `warnings.json` alone;
|
||||
2. there is no framework-level validation or aggregate bound, and the NPC- and
|
||||
spell-relatedness validators bypass the D&D warning limiter entirely;
|
||||
3. warnings returned by an output encoder are added after `warnings.json` has
|
||||
already been encoded, so the receipt, stderr, debug bundle, and published
|
||||
warning file can disagree; and
|
||||
4. a skipped validator contributes to `incomplete` validation but does not
|
||||
receive the warning generated for an exhausted validator failure.
|
||||
|
||||
The recommended end state is a structured diagnostic contract with explicit
|
||||
disposition, category, origin, occurrence count, and bounded samples. Warnings
|
||||
are reserved for process-level degradation or incompleteness. LLM-judged or
|
||||
deterministically inferred extraction-quality signals are advisories, never
|
||||
warnings, and routine normalization observations remain inspectable without
|
||||
being reported as top-level warnings. An ordinary successful run in which all
|
||||
configured work completes normally should therefore report zero warnings. This
|
||||
is an architectural and durable-contract change, not merely revised CLI prose.
|
||||
|
||||
## Evidence And Limits
|
||||
|
||||
The audit used:
|
||||
|
||||
- a complete static search of production `contracts.Warning` constructors,
|
||||
reason-code constants, result fields, and promotion sites under `internal/`;
|
||||
- call-path inspection through producer attempts, validators, lane
|
||||
coordination, chunk-plan reuse, checkpoints, output encoding, debug output,
|
||||
CLI presentation, and run-result construction;
|
||||
- the maintained complete and minimal D&D examples with offline fake LLMs;
|
||||
- focused deterministic tests for warning bounds, semantic-reconciliation
|
||||
fallback, `warn_continue`, semantic retries, terminal rejection, concurrency
|
||||
ordering, and checkpoint reuse; and
|
||||
- the operator-provided observation of an 85-warning complete D&D run.
|
||||
|
||||
No provider-backed production run was attempted because this environment has
|
||||
no API key. Consequently, the audit can establish warning mechanics, possible
|
||||
multiplicity, synthetic volume, and obvious heuristic limitations, but cannot
|
||||
estimate production frequency or the real false-positive rate of individual
|
||||
D&D advisories. Those measurements are not required to choose the recommended
|
||||
architecture; they are required before strengthening any heuristic advisory
|
||||
into a rejection or setting a numerical production acceptance target.
|
||||
|
||||
## Complete Warning-Producer Inventory
|
||||
|
||||
### Framework And Generic Boundaries
|
||||
|
||||
| Producer | Reason code | Trigger and consequence | Multiplicity and bound | Current surfaces and coverage |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Reference materialization in `internal/framework/pipeline/references.go` | `empty_reference` | A bound external reference is a valid, accepted media type but contains zero bytes. The prompt may receive materially incomplete context. | One per empty bound file; finite by configuration but no shared run-level cap. | Enters `RunInput.Warnings`; reference tests protect contextual scope. |
|
||||
| Producer-attempt policy in `internal/framework/pipeline/producer_attempts.go` | `validator_execution_incomplete` | An applicable validator exhausted its execution budget and `warn_continue` accepted the otherwise valid candidate. | One per failed validator on each terminal candidate. An extract chain can multiply this by chunks and lanes. There is no global cap. | Durable warning, receipt count, stderr, debug, and validation summary. `TestWarnContinueRecordsOneWarningForEachExhaustedValidator` covers failures. |
|
||||
| Chunk, extract, merge, normalize, and output module result contracts | Module-defined | A module may return arbitrary warnings with its successful candidate. | No contract validation, message limit, per-result cap, or global cap. Current production modules are inventoried below. | Terminal-attempt filtering and concurrency ordering are well tested. |
|
||||
| Production JSON output encoder | None | The encoder copies incoming warnings into `warnings.json`; it does not currently create warnings. | Same count as its input. | JSON encoder and assembled-pipeline tests compare the incoming run warnings with the published file. |
|
||||
| Output encoder result contract | Module-defined | Any output encoder may return warnings discovered during encoding. | Unbounded by contract. No production encoder currently exercises this capability. | Appended to final `RunOutput` only after logical files were encoded; this is the cross-surface defect described in AUD-WARN-004. |
|
||||
|
||||
Input adapters and production mergers do not currently have independent
|
||||
warning producers. Chunk-plan and checkpoint decisions are structured manifest
|
||||
or debug provenance rather than warnings. Cancellation and hard persistence,
|
||||
reference, parsing, serialization, and provider failures remain errors.
|
||||
|
||||
### D&D Extraction Gates
|
||||
|
||||
| Producer | Reason code | Trigger and consequence | Multiplicity and bound |
|
||||
| --- | --- | --- | --- |
|
||||
| `dnd/combat-turns` extractor | `scene_classification_unavailable` | The chunk has no exact matching scene-description classification. The extractor returns an empty accepted result and skips the LLM, so combat-turn output may be incomplete. | At most one per chunk for this lane. |
|
||||
| `dnd/enemy-events` extractor | `scene_classification_unavailable` | The same missing or mismatched scene gate causes accepted empty enemy-event output. | At most one per chunk for this lane. |
|
||||
|
||||
An exact non-combat classification produces an intentional empty result without
|
||||
a warning. An exact combat classification proceeds normally. The two producers
|
||||
share a code and operator consequence but use different messages; their module
|
||||
origins are not retained in the final warning record.
|
||||
|
||||
### D&D Source-Relatedness Validators
|
||||
|
||||
All ten relatedness validators are deterministic advisories: they approve the
|
||||
candidate and warn when contextual prose or an entity name is not lexically
|
||||
present in cited text. Shape and source-reference failures are deliberately
|
||||
left to blocking validators earlier in the chain. The same relatedness
|
||||
validator is registered in both the extract and normalize default chain for
|
||||
each artifact family in `internal/modules/dnd/register/chains.go`.
|
||||
|
||||
| Artifact family | Warning reason | Per-record trigger | Local bound | Omission reason |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Combat turns | `combat_turn_not_near_source` | Actor token sequence absent | 20 per validator invocation | `combat_turn_relatedness_warnings_omitted` |
|
||||
| Enemy events | `enemy_event_not_near_source` | Subject token sequence absent | 20 | `enemy_event_relatedness_warnings_omitted` |
|
||||
| Item occurrences | `item_occurrence_source_unrelated` | Item name token sequence absent | 20 | `item_occurrence_relatedness_warnings_omitted` |
|
||||
| Item registry | `item_not_near_source` | Item name token sequence absent | 20 | `item_relatedness_warnings_omitted` |
|
||||
| Location occurrences | `location_occurrence_not_near_source` | Location name token sequence absent | 20 | `location_occurrence_relatedness_warnings_omitted` |
|
||||
| Location registry | `location_not_near_source` | Location name token sequence absent | 20 | `location_relatedness_warnings_omitted` |
|
||||
| NPC occurrences | `npc_occurrence_not_near_source` | NPC name token sequence absent | 20 | `npc_occurrence_relatedness_warnings_omitted` |
|
||||
| NPC registry | `npc_not_near_source` | NPC name token sequence absent | **Unbounded** | None |
|
||||
| Scene descriptions | `scene_description_not_near_source` | No significant title or summary token appears; up to two findings per scene | 20 | `scene_description_relatedness_warnings_omitted` |
|
||||
| Spells | `spell_not_near_source` | Spell-name token sequence absent | **Unbounded** | None |
|
||||
|
||||
The eight limiter-generated omission records are presentation artifacts, not
|
||||
new source-relatedness conditions. They occupy a warning slot and make list
|
||||
length differ from the actual occurrence count.
|
||||
|
||||
### D&D Normalizers
|
||||
|
||||
Every production D&D normalizer bounds its returned warning slice to 20 through
|
||||
`internal/modules/dnd/shared/diagnostics`, including a final omission record
|
||||
when needed. Registry semantic retries reserve space for their fallback
|
||||
warning. The following table is complete by semantically distinct condition;
|
||||
codes listed together are parallel artifact-family variants.
|
||||
|
||||
| Condition | Reason codes | Result impact | Current classification assessment |
|
||||
| --- | --- | --- | --- |
|
||||
| Display or field whitespace/name canonicalization | `npc_fields_normalized`, `item_fields_normalized`, `location_fields_normalized`, `spell_name_canonicalized`, `combat_actor_canonicalized`, `enemy_event_name_canonicalized`, `item_occurrence_name_canonicalized`, `location_occurrence_name_canonicalized`, `scene_description_prose_normalized` | Deterministic successful mutation. The item-occurrence code can also describe `from`/`to` whitespace, not only the item name. | Routine observation. |
|
||||
| Durable ID recomputation | `npc_id_recomputed`, `item_id_recomputed`, `location_id_recomputed` | Restores the deterministic name-derived ID. | Routine observation; invalid identity is separately rejected by default chains. |
|
||||
| Source-reference sorting or deduplication | `source_references_normalized` | Sorts and removes exact duplicate references while deliberately preserving invalid references for their validators. | Routine observation. Shared code is useful but ambiguous without producer origin. |
|
||||
| Canonical record ordering | `combat_turns_reordered`, `enemy_events_reordered`, `item_occurrences_reordered`, `location_occurrences_reordered`, `npc_occurrences_reordered`, `scene_description_order_normalized` | Deterministic order changes only. | Routine observation. |
|
||||
| Exact or approved semantic duplicate consolidation | `duplicate_npc_collapsed`, `duplicate_item_collapsed`, `duplicate_location_collapsed`, `duplicate_spell_cast_collapsed`, `duplicate_combat_turn_collapsed`, `duplicate_enemy_event_collapsed`, `duplicate_item_occurrence_collapsed`, `duplicate_location_occurrence_collapsed`, `duplicate_npc_occurrence_collapsed`, `scene_description_duplicate_collapsed` | Removes duplicate records and preserves or combines canonical evidence according to the artifact policy. Registry codes cover both exact and accepted semantic consolidation. | Durable normalization observation; not normally operator-actionable. |
|
||||
| Unresolved external membership | `spell_name_unresolved`, `item_occurrence_unknown_item_id`, `location_occurrence_unknown_location_id` | The value is preserved but is not grounded in the effective catalog or registry. Default chains normally reject the same condition before normalization; it remains reachable with validator overrides or defensive direct use. | Actionable data-quality warning. |
|
||||
| Unsafe currency consolidation proposal | `item_semantic_proposal_invalid` | The proposed group is rejected and all records are preserved because denominations or currency/non-currency members are incompatible. The same code is also used internally as a retry reason. | Advisory about model proposal quality; no accepted-data loss. The control and diagnostic meanings should be separated. |
|
||||
| Semantic reconciliation unavailable or exhausted | `npc_semantic_reconciliation_exhausted`, `item_semantic_reconciliation_exhausted`, `location_semantic_reconciliation_exhausted` | The safe deterministic result is accepted, but possible semantic duplicates remain. | Actionable fallback warning. |
|
||||
| Local warning truncation | `npc_normalization_warnings_omitted`, `item_normalization_warnings_omitted`, `location_normalization_warnings_omitted`, `spell_normalization_warnings_omitted`, `combat_turn_normalization_warnings_omitted`, `enemy_event_normalization_warnings_omitted`, `item_occurrence_normalization_warnings_omitted`, `location_occurrence_normalization_warnings_omitted`, `npc_occurrence_normalization_warnings_omitted`, `scene_description_normalization_warnings_omitted` | Reports that individual records were omitted from presentation. | Group metadata, not an independent warning. |
|
||||
|
||||
No production D&D normalization warning exposes raw model responses,
|
||||
correction guidance, or provider errors. Most dynamic names are quoted and
|
||||
truncated by the shared helper. That local discipline is not enforced by the
|
||||
generic warning contract, and the spell relatedness message does not use the
|
||||
shared truncation helper.
|
||||
|
||||
## Warning Propagation And Surface Map
|
||||
|
||||
```text
|
||||
external-reference warnings -----------------------------+
|
||||
|
|
||||
module candidate warnings -> validation chain warnings |
|
||||
| | |
|
||||
+---- producer-attempt terminal policy -------+
|
||||
| |
|
||||
accepted / terminal rejection only |
|
||||
| |
|
||||
chunk or lane result in canonical order |
|
||||
| |
|
||||
checkpoint record/replay and ordered step merge |
|
||||
| |
|
||||
RunOutput.Warnings <------------+
|
||||
|
|
||||
OutputRequest -> output encoder
|
||||
| |
|
||||
warnings.json OutputResult.Warnings
|
||||
|
|
||||
appended to final RunOutput only
|
||||
|
|
||||
receipt, stderr, final debug warning summary
|
||||
```
|
||||
|
||||
### Attempts And Validation
|
||||
|
||||
- `runProducerAttempts` promotes only the terminal accepted or terminal
|
||||
rejected candidate's module and completed-validator warnings. Operational,
|
||||
structural, semantic, and module-directed attempts that are superseded are
|
||||
retained in attempt debug artifacts but not in the final collection.
|
||||
- A module-directed semantic-reconciliation retry adds its fallback warning
|
||||
only when no retry remains. Earlier attempt warnings are discarded.
|
||||
- On `warn_continue`, warnings from the otherwise accepted candidate and
|
||||
completed approved or rejected validators are retained. One fixed,
|
||||
non-sensitive `validator_execution_incomplete` warning is added for every
|
||||
failed validator. Skipped validators affect the validation summary and final
|
||||
`incomplete` status but do not receive such a warning.
|
||||
- A semantic terminal rejection retains only warnings from that rejected
|
||||
attempt. Structural rejection after producer failure cannot retain a
|
||||
candidate warning because no valid candidate result exists.
|
||||
|
||||
These behaviors are protected by the producer-attempt, extract-handoff,
|
||||
rejection-warning, normalize-retry, and attempt-debug tests. They are the right
|
||||
foundation for the redesign and should not be replaced with early-exit or
|
||||
all-attempt accumulation.
|
||||
|
||||
### Concurrency And Ordering
|
||||
|
||||
Extract jobs are dispatched chunk-first and lane-second. Results are stored by
|
||||
chunk index, finalized in ascending chunk order, and lane continuations are
|
||||
merged into a slice indexed by configured lane order. Pipeline steps run in
|
||||
configured order. The resulting public order is therefore:
|
||||
|
||||
1. pre-run reference warnings;
|
||||
2. chunk-stage warnings;
|
||||
3. step order;
|
||||
4. configured lane order within each step;
|
||||
5. chunk order within extract;
|
||||
6. merge warnings; then
|
||||
7. normalize warnings; followed by any output-result warnings.
|
||||
|
||||
`TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion` exercises warning
|
||||
order under reversed completion. No completion-order leak was found.
|
||||
|
||||
### Chunk Plans And Checkpoints
|
||||
|
||||
- A reusable chunk plan stores producer warnings only. Current validators run
|
||||
again, and their current warnings are appended. Warnings from a cached plan
|
||||
candidate that fails current validation are discarded before regeneration.
|
||||
- Accepted extract, merge, and normalize checkpoints store the terminal
|
||||
warnings for their stage. Reuse loads and appends those warnings once at the
|
||||
same logical handoff. Tests compare fresh and resumed warning collections and
|
||||
preserve their order.
|
||||
- Validation-incomplete accepted outputs are not reusable, preventing a later
|
||||
run from silently treating incomplete validation as complete.
|
||||
- Required accepted-normalize hydration replays that normalize checkpoint's
|
||||
warnings; checkpoint decisions separately expose that reuse occurred.
|
||||
|
||||
The recommended redesign should keep fresh and resumed logical diagnostics
|
||||
equivalent. Whether a result was reused belongs in checkpoint provenance, not
|
||||
in the diagnostic grouping key; adding a `reused` distinction would fragment
|
||||
groups and make equivalent runs present differently.
|
||||
|
||||
### Terminal Surfaces
|
||||
|
||||
| Surface | Current content | Audience | Audit result |
|
||||
| --- | --- | --- | --- |
|
||||
| `RunOutput.Warnings` | Flat final slice | Framework and CLI | Canonical in-memory list, but lacks origin and bounds. |
|
||||
| Published `warnings.json` | Object containing the warnings passed into the output encoder | Durable consumers | Exact for the production JSON encoder unless the encoder itself returns warnings. |
|
||||
| `index.json` | Path to `warnings.json` | Durable consumers | Stable discovery path; no separate diagnostic-detail path. |
|
||||
| Run-result v1 | `warning_count = len(final RunOutput.Warnings)` | Subprocess callers | Count only; no group/occurrence distinction. |
|
||||
| Human stderr | `run completed with N warning(s)` | Operators | Count only and no direct detail path. Successful exit remains zero. |
|
||||
| Manifest | Validation and rejection summaries, no warning collection | Durable provenance | Correctly avoids duplicating the flat list. |
|
||||
| Debug summary `warnings.json` | Raw final warning array | Operators/developers | Includes final output-result warnings and can therefore differ from published `warnings.json`. |
|
||||
| Debug run report | Final warning count | Operators/developers | Same final slice length as receipt and stderr. |
|
||||
| Attempt/stage debug | Candidate-local warning detail and origin in path/envelope | Forensics | Sufficient to diagnose provenance, but debug capture is optional and is not a durable consumer contract. |
|
||||
|
||||
## Empirical Measurements
|
||||
|
||||
### Offline And Synthetic Runs
|
||||
|
||||
| Scenario | Result | What it establishes |
|
||||
| --- | --- | --- |
|
||||
| Maintained complete D&D config and transcript with the repository's offline fake LLM | Approved, 10 normalized outputs, 0 rejected outputs, 12 warnings; receipt, stderr, and published file all reported 12 | An ordinary structurally successful workflow can be noisy without fallback or incomplete validation. |
|
||||
| Same complete run, grouped after publication | Three reason codes, seven exact `(reason, scope, message)` tuples, maximum exact-tuple repetition of three | The flat count materially overstates distinct operator conditions. Scope resets within chunks and does not identify origin. |
|
||||
| Maintained focused scene-description workflow | Approved, one normalized output, 0 warnings | The warning channel can be quiet when synthetic model text is lexically grounded. |
|
||||
| Generic warning publication contract | One warning reaches successful stderr, durable output, and debug summary | The ordinary pre-output path is consistent. |
|
||||
| NPC semantic-reconciliation candidate-limit fallback | No LLM call, all records preserved, one exhaustion warning, total warnings no greater than 20 | Fallback is bounded and materially different from routine normalization. |
|
||||
| `warn_continue` with two failed validators and one skipped validator | Validation status contains all three incomplete validators; warning slice contains two execution-incomplete records | Current warning count does not describe all incomplete validation. |
|
||||
| Retrying extract candidate | Two producer attempts; only the accepted attempt's one warning is final | Retry does not amplify abandoned warnings. |
|
||||
| Terminal semantic rejection | Only the final rejected attempt's operation and validator warnings are final | Rejection diagnostics are retained without retaining superseded warnings. |
|
||||
| Fresh versus reused extract checkpoint | Warning collections are deeply equal | Checkpoint replay does not itself amplify warnings. |
|
||||
| Spell normalizer with 21 unresolved entries | 20 records: 19 samples plus one omission record saying two additional warnings were omitted | `warning_count` is neither exact occurrence count nor distinct-condition count. |
|
||||
|
||||
The focused audit tests passed in `internal/cli`,
|
||||
`internal/framework/pipeline`, the NPC-registry and spell normalizers, and all
|
||||
D&D packages.
|
||||
|
||||
### Bounded Sample Review
|
||||
|
||||
The complete offline D&D run produced:
|
||||
|
||||
| Reason | Count | Sample | Review |
|
||||
| --- | ---: | --- | --- |
|
||||
| `location_not_near_source` | 4 | `Moon Gate` was absent from cited text | Correctly identifies deliberately unsupported fake output. Three records shared the same exact tuple because chunk and stage origin were lost. |
|
||||
| `location_occurrence_not_near_source` | 4 | A `Moon Gate` visit was absent from cited text | Correctly identifies the same unsupported registry-driven occurrence, but repeats the same operator concern across extraction and normalization. |
|
||||
| `scene_description_not_near_source` | 4 | A title or summary had no significant exact token in cited text | Mixed value. Generic `session scene` prose is ungrounded, while `Arrival` versus transcript `arrive` illustrates an expected lexical false positive. |
|
||||
|
||||
This fake workflow is an integration fixture, not a model-quality benchmark.
|
||||
It nonetheless proves that the checks carry useful evidence while being too
|
||||
imprecise and repetitive to serve as one-warning-per-record operator alerts.
|
||||
|
||||
### Production Evidence Still Needed
|
||||
|
||||
The reported 85-warning run establishes that high volume occurs in practice,
|
||||
but the following remain unknown:
|
||||
|
||||
- dominant production reason codes and stage/lane sources;
|
||||
- unique group count versus repeated occurrence count;
|
||||
- false-positive rate for each relatedness family;
|
||||
- how much volume comes from normalization observations versus advisories;
|
||||
- whether fresh and resumed production runs remain equivalent; and
|
||||
- a defensible numerical acceptance target.
|
||||
|
||||
If further data is worthwhile, the operator can supply the v1 receipt,
|
||||
`warnings.json`, and manifest validation summaries without supplying transcript
|
||||
or lane artifacts. An initial privacy-preserving report should group by reason
|
||||
code and normalized scope family, count exact repeated tuples, and omit message
|
||||
text. Reviewing heuristic precision requires a separately approved bounded
|
||||
sample with its cited source context.
|
||||
|
||||
## Classification Of Current Conditions
|
||||
|
||||
| Target disposition | Current families | Result impact | Operator action | Durable placement |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| **Warning** | Empty reference; validator failure or skip accepted under `warn_continue`; unavailable required scene classification; exhausted semantic reconciliation | A configured process completed under an allowed degraded or incomplete policy rather than completing normally | Correct reference/configuration, inspect provider/validator, or rerun | Actionable `warnings.json`, receipt summary, stderr summary, debug |
|
||||
| **Advisory** | Source-relatedness heuristics; unresolved spell or registry membership; guarded invalid semantic proposal; any future LLM-judged uncertainty or extraction-quality signal | Uncertain data quality or poor model proposal, but accepted data is structurally valid and deterministic guards prevented unsafe mutation | Optional model/source review; no routine action for every record | Durable diagnostic detail and debug; never a top-level warning |
|
||||
| **Observation** | Whitespace/name/ID/source-reference canonicalization; canonical ordering; exact and approved semantic duplicate consolidation | Successful intended normalization | None under normal operation | Durable bounded normalization diagnostics or debug; no stderr warning |
|
||||
| **Not a diagnostic** | Rejection, invalid structure, cancellation, persistence error, provider failure under fail-run policy | Candidate or run did not complete according to policy | Inspect rejection/error and retry or correct input/configuration | Existing rejection, validation summary, error, and debug contracts |
|
||||
|
||||
Exact and semantic duplicate consolidation should remain distinguishable in
|
||||
category or reason metadata even though both are observations. Semantic
|
||||
reconciliation exhaustion remains a warning because a capability was not
|
||||
applied; successful approved consolidation is an observation because it is the
|
||||
normalizer's intended work.
|
||||
|
||||
## Ranked Findings
|
||||
|
||||
### AUD-WARN-001 — The Flat Warning Type Destroys Signal Quality
|
||||
|
||||
- **Priority:** High operator impact; high implementation leverage.
|
||||
- **Evidence:** `contracts.Warning` has only scope, reason, and message. Routine
|
||||
normalizer changes, heuristic doubt, fallback, and incomplete validation all
|
||||
enter the same slice and the same CLI count. The offline complete run's 12
|
||||
records were all advisories; the operator observed 85 records in a successful
|
||||
real run.
|
||||
- **Impact:** Operators cannot tell whether a warning requires a rerun, a
|
||||
configuration repair, optional review, or no action. Repeated routine output
|
||||
trains them to ignore the channel.
|
||||
- **Recommendation:** Replace the flat semantic contract with explicit
|
||||
`warning`, `advisory`, and `observation` dispositions plus a small category
|
||||
vocabulary. Do not infer disposition from message text or require every
|
||||
downstream consumer to maintain a reason-code policy table.
|
||||
|
||||
### AUD-WARN-002 — Warning Records Lose The Origin Needed For Diagnosis And Aggregation
|
||||
|
||||
- **Priority:** High correctness and usability impact.
|
||||
- **Evidence:** The runner knows stage, step, lane, module, validator, chunk ID,
|
||||
and chunk index at promotion time, but `terminalWarnings` flattens module and
|
||||
validator records into `[]contracts.Warning`. Per-chunk scopes such as
|
||||
`locations[0]` and `occurrences[0]` then repeat without identifying their
|
||||
chunk or producer. `source_references_normalized` is intentionally shared
|
||||
across families and is therefore especially ambiguous.
|
||||
- **Impact:** `warnings.json` cannot answer which stage or module produced a
|
||||
record. Message- or scope-based deduplication would merge unrelated findings
|
||||
or retain accidental duplicates.
|
||||
- **Recommendation:** Keep module findings free of framework context, then have
|
||||
the framework add a structured origin envelope before promotion. Validator
|
||||
findings must retain validator identity instead of passing through
|
||||
`validationReport.Warnings()` as a flat slice.
|
||||
|
||||
### AUD-WARN-003 — Warning Volume Is Not End-To-End Bounded Or Validated
|
||||
|
||||
- **Priority:** High robustness impact; medium immediate likelihood.
|
||||
- **Evidence:** D&D's `LimitWarnings` caps most individual producers at 20, but
|
||||
NPC- and spell-relatedness return one warning per record without the helper.
|
||||
Every extract validator is invoked per chunk, all ten relatedness checks run
|
||||
again after normalization, and there is no run-level collector. The generic
|
||||
contract validates neither disposition nor reason/message size, UTF-8,
|
||||
blankness, or total records.
|
||||
- **Impact:** Warning memory and output grow with chunks, lanes, configured
|
||||
validators, and record counts. Local omission records lose exact occurrence
|
||||
semantics while still incrementing `warning_count`.
|
||||
- **Recommendation:** Add a generic bounded diagnostic collector that preserves
|
||||
exact occurrence counts and bounded samples. Validate all diagnostic fields
|
||||
at the module/framework boundary. Immediately bring NPC and spell
|
||||
relatedness under the existing cap if the full redesign is staged.
|
||||
|
||||
### AUD-WARN-004 — Output Encoder Warnings Make Durable Surfaces Disagree
|
||||
|
||||
- **Priority:** Medium current impact; high contract correctness risk.
|
||||
- **Evidence:** `Runner.Run` passes existing warnings to `encoder.Encode`, then
|
||||
the production encoder serializes `warnings.json`. Only after encoding does
|
||||
the runner append `OutputResult.Warnings`. The receipt, stderr, debug summary,
|
||||
and debug run report see the final slice; the already-created published file
|
||||
cannot. No production encoder currently returns a warning, so ordinary JSON
|
||||
runs do not trigger the defect.
|
||||
- **Impact:** A valid output-module implementation can violate the documented
|
||||
claim that `warning_count` describes the published warning collection.
|
||||
- **Recommendation:** Remove successful output warnings from the output-module
|
||||
contract unless a demonstrated use case requires them; encoding failures
|
||||
should be errors and optional encoder observations should be debug data. A
|
||||
two-phase finalize API is the viable but more complex alternative.
|
||||
|
||||
### AUD-WARN-005 — Validation Skips Are Incomplete But Not Warned
|
||||
|
||||
- **Priority:** Medium operator/correctness impact.
|
||||
- **Evidence:** `firstIncompleteValidation` treats failed and skipped validators
|
||||
alike, and validation summaries include both. `incompleteValidationWarnings`
|
||||
emits records only for `validationFailed`. The focused test demonstrates
|
||||
three incomplete validators but two warnings.
|
||||
- **Impact:** A successful run can have `validation_status: incomplete` while
|
||||
its warning count understates or even omits the affected validators. A caller
|
||||
that checks only warnings receives a weaker signal than the manifest and
|
||||
receipt status.
|
||||
- **Recommendation:** Produce one aggregated incomplete-validation warning
|
||||
group whose occurrences cover both failure and skip, while retaining typed
|
||||
outcome and safe reason metadata in the validation summary. Do not expose
|
||||
provider errors or arbitrary skip prose in model or operator messages.
|
||||
|
||||
### AUD-WARN-006 — Relatedness Checks Are Useful But Repetitive And Lexically Weak
|
||||
|
||||
- **Priority:** Medium operator impact; low acceptance-policy urgency.
|
||||
- **Evidence:** Every family runs the advisory in both extract and normalize
|
||||
chains. The complete fixture contains exact repeated tuples, and the checks
|
||||
rely on exact normalized token sequences or a minimal significant-token
|
||||
overlap. Reason naming drifts between `*_not_near_source` and
|
||||
`*_source_unrelated`.
|
||||
- **Impact:** The checks can catch unsupported entities, but aliases, pronouns,
|
||||
inflection, and generic scene prose create predictable false positives. Flat
|
||||
per-record presentation magnifies them.
|
||||
- **Recommendation:** Retain the validators and their stage-local execution,
|
||||
but classify and aggregate them as advisories. Normalize reason-code naming
|
||||
when the diagnostic contract changes. Do not strengthen them into rejection
|
||||
rules without a human-reviewed production evaluation.
|
||||
|
||||
### AUD-WARN-007 — `warning_count` Has No Stable Operational Meaning
|
||||
|
||||
- **Priority:** High downstream-contract impact.
|
||||
- **Evidence:** The receipt and CLI use `len(output.Warnings)`. One list element
|
||||
can be an omission summary representing several hidden occurrences; repeated
|
||||
records can represent the same condition; and skipped validators can be
|
||||
absent. A 21-occurrence spell test produces a list length of 20.
|
||||
- **Impact:** The value is neither an exact occurrence count nor a distinct
|
||||
warning-group count. Consumers cannot set policy or present a trustworthy
|
||||
summary from it.
|
||||
- **Recommendation:** Introduce explicit warning-group and warning-occurrence
|
||||
counts in a versioned receipt. Do not silently redefine the v1 field.
|
||||
|
||||
## Recommended Target Contract And Presentation Model
|
||||
|
||||
### Diagnostic Model
|
||||
|
||||
Use one validated internal diagnostic model with these concepts:
|
||||
|
||||
- **disposition:** `warning`, `advisory`, or `observation`;
|
||||
- **category:** a small enum such as `configuration`, `degradation`,
|
||||
`validation_incomplete`, `data_quality`, `fallback`, or `normalization`;
|
||||
- **reason code:** stable semantic identity owned by the producer;
|
||||
- **origin:** framework-added phase/stage, step ID, lane ID, module key,
|
||||
validator name, chunk ID, and chunk index when applicable;
|
||||
- **occurrence count:** exact number of matching findings;
|
||||
- **samples:** a small deterministic list of bounded scope/message pairs; and
|
||||
- **omitted sample count:** `occurrence_count - len(samples)`, represented as
|
||||
metadata rather than another diagnostic record.
|
||||
|
||||
Errors and rejected outputs must not become diagnostic dispositions. A warning
|
||||
means that the run completed under policy despite a process-level degradation
|
||||
or incomplete configured operation. Advisory and observation dispositions can
|
||||
describe accepted artifact quality and transformation provenance, but no
|
||||
LLM-judged extraction-quality signal may be promoted to a warning. Validation
|
||||
status remains authoritative for approval, rejection, and incomplete
|
||||
validation.
|
||||
|
||||
### Aggregation
|
||||
|
||||
The framework runner should own aggregation after it enriches findings with
|
||||
origin and before public output construction. Modules and validators retain
|
||||
semantic ownership of disposition, category, reason, scope, and message; they
|
||||
must not own CLI or file presentation.
|
||||
|
||||
The default stable key should be:
|
||||
|
||||
```text
|
||||
disposition + category + reason_code
|
||||
+ phase/stage + step_id + lane_id + module_key + validator_name
|
||||
```
|
||||
|
||||
Chunk, record scope, and message text belong in samples and must not be part of
|
||||
the group key. This groups repeated per-chunk findings without merging the same
|
||||
code across distinct producers or pipeline locations. Group order should be
|
||||
the first occurrence in the runner's existing canonical order; sample order
|
||||
should follow the same order. A final canonical sort by the complete origin key
|
||||
is also viable, but completion timing must never choose either order.
|
||||
|
||||
Aggregation must be incremental and bounded. Producers should use a shared
|
||||
collector that counts every occurrence while retaining only bounded samples;
|
||||
the framework then merges producer groups without reconstructing counts from
|
||||
omission prose. A global maximum group count is also required, with overflow
|
||||
represented by structured aggregate metadata and with actionable groups given
|
||||
priority over lower dispositions.
|
||||
|
||||
### Durable Files
|
||||
|
||||
Keep one canonical home for each class:
|
||||
|
||||
- `warnings.json` should contain versioned, grouped actionable warnings only;
|
||||
- a new `diagnostics.json` should contain versioned advisory and observation
|
||||
groups only, avoiding duplication of warning groups;
|
||||
- `index.json` should link both files;
|
||||
- `rejected.json` and manifest validation summaries should retain their current
|
||||
separate responsibilities; and
|
||||
- debug bundles should retain candidate-attempt detail plus the final grouped
|
||||
projections.
|
||||
|
||||
This is preferable to keeping all detail in `warnings.json` and filtering only
|
||||
the CLI: downstream consumers would otherwise continue to receive a semantically
|
||||
mixed warning contract, and routine observations would still dominate the
|
||||
durable file.
|
||||
|
||||
### CLI And Receipt
|
||||
|
||||
For a successful human run with actionable warnings, print a concise summary
|
||||
such as:
|
||||
|
||||
```text
|
||||
notarius: run completed with 2 warning groups (7 occurrences); details=/.../warnings.json
|
||||
```
|
||||
|
||||
Advisories and observations should not produce the warning line. Their durable
|
||||
path remains discoverable through `index.json`; a concise non-warning count can
|
||||
be added to the ordinary success line only if operator testing shows value. An
|
||||
ordinary successful run with no process degradation should write nothing to
|
||||
the warning stream even when it publishes quality advisories or normalization
|
||||
observations.
|
||||
|
||||
Create `notarius.run-result.v2` rather than redefining v1. It should expose at
|
||||
least:
|
||||
|
||||
- `warning_group_count`;
|
||||
- `warning_occurrence_count`; and
|
||||
- `diagnostic_group_count` for non-warning durable groups.
|
||||
|
||||
The receipt should continue to expose validation status, validation summaries,
|
||||
and rejected-output count independently. Process exit behavior should not
|
||||
change as part of warning presentation reform.
|
||||
|
||||
### Checkpoint Semantics
|
||||
|
||||
Store the structured terminal diagnostic groups with accepted checkpoints and
|
||||
replay them exactly once at their logical stage. Fresh and reused runs should
|
||||
produce the same public groups and counts. Checkpoint events and debug records,
|
||||
not diagnostic identity, should disclose whether computation was reused.
|
||||
|
||||
### Output Encoder Boundary
|
||||
|
||||
Prefer removing `OutputResult.Warnings`. A successful output encoder should
|
||||
either return the complete logical files or fail. If future encoders genuinely
|
||||
need to produce durable post-encoding warnings, introduce an explicit
|
||||
two-phase prepare/finalize contract so those warnings can be included in the
|
||||
same published collection. Do not retain the current self-inconsistent
|
||||
one-pass capability.
|
||||
|
||||
## Resolution Of Required Design Questions
|
||||
|
||||
| Question | Recommendation | Viable alternative and tradeoff |
|
||||
| --- | --- | --- |
|
||||
| Explicit severity/disposition or external reason mapping? | Put validated disposition and category in the contract. | A central reason-code registry avoids payload fields but makes new modules depend on a second synchronized policy table and leaves downstream meaning implicit. |
|
||||
| Keep all detail in `warnings.json` or separate it? | Separate grouped actionable warnings from grouped advisories/observations in `diagnostics.json`. | Keep the flat durable list and aggregate only CLI output; simpler migration, but it preserves the noisy downstream contract and ambiguous count. |
|
||||
| Who owns aggregation? | Framework runner/coordinator after origin enrichment. | Output module aggregation keeps framework types smaller but duplicates policy across encoders and cannot repair missing validator origin. |
|
||||
| Stable aggregation key? | Disposition, category, reason code, and full producer origin; exclude chunk/scope/message. | Explicit producer-supplied grouping keys offer flexibility but add another identity that can drift from reason codes. Message-template grouping is brittle and unsafe. |
|
||||
| Samples and omissions? | Exact occurrence count plus deterministic bounded samples and numeric omitted-sample count. | Omission warning records preserve the current representation but inflate group counts and require prose parsing. |
|
||||
| `warning_count` semantics? | Version receipt and replace ambiguity with group and occurrence counts. | Keep v1 count as published record length and add optional fields; compatible, but two competing warning counts remain easy to misuse. |
|
||||
| Which normalization changes remain warnings? | Only exhausted process fallback. Unresolved membership is a data-quality advisory; successful canonicalization, reordering, ID repair, source-ref dedupe, and duplicate consolidation are observations. | Treat unresolved membership or semantic duplicate consolidation as warnings because they affect grounding or cardinality; more conservative, but it violates the process-only warning rule and reports accepted artifact quality as an operational failure. |
|
||||
| Source-relatedness disposition? | Grouped advisory by default; preserve current approve behavior. | Retain warning disposition or make rejection configurable. Rejection requires production precision evidence; current lexical rules are not strong enough. |
|
||||
| Checkpoint-loaded warnings? | Present the same logical groups as fresh execution and use checkpoint events for reuse provenance. | Mark groups as replayed; aids forensics but fragments aggregation and makes semantically equivalent runs differ. |
|
||||
| ADR and schema versions? | Add an ADR and version the run receipt and diagnostic files. | Treat the work as CLI-only presentation and avoid an ADR; insufficient because module contracts, output files, checkpoint payloads, and downstream fields change. |
|
||||
|
||||
## Compatibility, Documentation, And ADR Implications
|
||||
|
||||
The target alters public and internal contracts enough to require a new ADR.
|
||||
It should record:
|
||||
|
||||
- the distinction among warnings, advisories, observations, rejections, and
|
||||
errors;
|
||||
- the invariant that warnings are process-level signals, LLM-judged extraction
|
||||
quality is never a warning, and ordinary non-degraded success has zero
|
||||
warnings;
|
||||
- module semantic ownership versus framework origin/aggregation ownership;
|
||||
- bounded group and sample semantics;
|
||||
- fresh/checkpoint equivalence; and
|
||||
- the output-encoder decision.
|
||||
|
||||
Implementation should introduce `notarius.run-result.v2`. The grouped warning
|
||||
and diagnostic envelopes should each carry their own schema version. Because
|
||||
the content of `warnings.json` changes incompatibly from a flat array wrapper
|
||||
to groups, release notes and the published JSON integration contract must call
|
||||
out the migration. `index.json` gains the diagnostic file path.
|
||||
|
||||
Canonical documentation updates belong in:
|
||||
|
||||
- `docs/cli.md` for stderr presentation only;
|
||||
- `docs/operations.md` for operator review and debug workflow;
|
||||
- `docs/integrations/json-output.md` for warning and diagnostic file schemas;
|
||||
- `docs/integrations/run-result.md` for v2 fields and compatibility;
|
||||
- `docs/consumers/subprocess.md` and `docs/consumers/dnd-pipeline.md` for
|
||||
downstream policy checks;
|
||||
- `docs/internal/pipeline.md` for promotion, aggregation, retry, and checkpoint
|
||||
mechanics;
|
||||
- `docs/internal/modules.md` and `docs/internal/dnd.md` for producer rules and
|
||||
the D&D classification matrix; and
|
||||
- `docs/policy/architecture.md` for the durable ownership invariant after the
|
||||
ADR is accepted and implemented.
|
||||
|
||||
No configuration knob is required for the first implementation. A fixed,
|
||||
well-documented taxonomy is easier to reason about than per-reason display
|
||||
overrides. Configurable escalation or suppression can be considered only after
|
||||
production review demonstrates a concrete operator need.
|
||||
|
||||
## Test Coverage Assessment
|
||||
|
||||
Existing coverage worth preserving includes:
|
||||
|
||||
- accepted-attempt and terminal-rejection warning promotion;
|
||||
- validator failure retry exhaustion and `warn_continue`;
|
||||
- module semantic retry fallback;
|
||||
- deterministic warning order under concurrent lane completion;
|
||||
- chunk-plan invalidation and discarded-cache warning behavior;
|
||||
- fresh/checkpoint warning equivalence;
|
||||
- local D&D warning caps and safe dynamic-message quoting;
|
||||
- JSON warning-file publication; and
|
||||
- CLI stderr, debug, and receipt counts.
|
||||
|
||||
Material gaps are:
|
||||
|
||||
- no bound test for NPC- or spell-relatedness warnings;
|
||||
- no generic warning-field or result-size validation;
|
||||
- no test for output encoder warnings versus published `warnings.json`;
|
||||
- no operator-level aggregation or bounded-sample tests;
|
||||
- no fresh/resume test for grouped counts because groups do not yet exist; and
|
||||
- no production evaluation of advisory precision.
|
||||
|
||||
Tests should protect the semantic relationships: exact occurrence counts,
|
||||
bounded samples, deterministic group order, actionable-only warning
|
||||
presentation, and cross-surface equality. They should not assert one exact
|
||||
warning count for every complete D&D run or treat message wording as a public
|
||||
API unless the wording itself enforces a security boundary.
|
||||
|
||||
## Audit Conclusion
|
||||
|
||||
The application is in a good position for warning reform. Its retry,
|
||||
validation, checkpoint, and concurrency mechanics provide reliable points at
|
||||
which to attach structured diagnostics. The most valuable change is not to
|
||||
suppress individual reason codes; it is to replace the semantically flat,
|
||||
origin-free collection with bounded typed groups and to reserve the word
|
||||
“warning” for conditions that merit operator attention.
|
||||
|
||||
Provider-backed runs would improve prioritization and help tune the D&D
|
||||
advisories, but they are not necessary to conclude that routine normalization
|
||||
and heuristic doubt should not dominate stderr or the durable warning
|
||||
contract. They should be gathered before changing heuristic acceptance policy
|
||||
or adopting a numerical production warning-volume target.
|
||||
@@ -1,224 +0,0 @@
|
||||
# D&D Subprocess Consumer Documentation
|
||||
|
||||
## Status
|
||||
|
||||
Completed. The target guide is `docs/consumers/dnd-pipeline.md`.
|
||||
|
||||
## Purpose
|
||||
|
||||
Provide one task-oriented guide for applications that run Notarius as a
|
||||
subprocess to execute the maintained complete D&D pipeline and consume its
|
||||
published artifacts. The initial concrete consumer is Narratio, but the guide
|
||||
must describe the public Notarius workflow rather than depend on Narratio
|
||||
internals.
|
||||
|
||||
The guide should make the safe integration path obvious without duplicating
|
||||
the CLI, input, receipt, output-bundle, or individual artifact contracts that
|
||||
already have canonical documentation.
|
||||
|
||||
## Current State
|
||||
|
||||
The public integration surface is documented accurately but is distributed
|
||||
across several documents:
|
||||
|
||||
- `docs/consumers/subprocess.md` defines the generic subprocess workflow;
|
||||
- `docs/cli.md` owns commands, flags, stream behavior, and exit statuses;
|
||||
- `docs/integrations/seriatim.md` owns the accepted transcript input shape;
|
||||
- `docs/integrations/run-result.md` owns the machine-readable successful-run
|
||||
receipt;
|
||||
- `docs/integrations/json-output.md` owns bundle discovery and logical files;
|
||||
- the D&D integration documents own the individual lane payload contracts;
|
||||
- `examples/dnd-complete.config.yml` is the maintained complete pipeline.
|
||||
|
||||
A consumer can reconstruct the full workflow from those documents, but there
|
||||
is no D&D-focused guide that connects the maintained example to its input,
|
||||
invocation, complete artifact inventory, discovery procedure, and downstream
|
||||
acceptance decisions.
|
||||
|
||||
## Target Documentation Set
|
||||
|
||||
### Create `docs/consumers/dnd-pipeline.md`
|
||||
|
||||
This document should own the end-to-end consumer workflow for the maintained
|
||||
complete D&D configuration. It should be useful to Narratio and to another
|
||||
subprocess orchestrator with the same needs.
|
||||
|
||||
The guide should contain the following sections.
|
||||
|
||||
#### Prerequisites And Deployment Configuration
|
||||
|
||||
- Link to `examples/dnd-complete.config.yml` rather than embedding a second
|
||||
complete configuration.
|
||||
- Explain that a deployment must provide the configured PromptKit profile and
|
||||
campaign reference files.
|
||||
- Recommend absolute paths for a service or orchestrator deployment.
|
||||
- Call out the path-resolution distinction explicitly: YAML reference paths
|
||||
are relative to the Notarius configuration file, while
|
||||
`promptkit.profile_file` is relative to the Notarius process working
|
||||
directory.
|
||||
- Recommend validating the selected configuration and `dnd-session` pipeline
|
||||
before processing sessions.
|
||||
|
||||
#### Transcript Input
|
||||
|
||||
- State that the complete pipeline consumes a Seriatim JSON document.
|
||||
- Link to the canonical Seriatim contract for required fields and validation.
|
||||
- Recommend the caller's final trimmed transcript when the caller maintains
|
||||
transcript tiers. For Narratio, identify the implemented source as
|
||||
`narratio.transcript.final_trimmed`, normally stored at
|
||||
`transcripts/final.trimmed.json`.
|
||||
- Explain that segment IDs must remain stable because D&D source references
|
||||
cite those units.
|
||||
- Explain that Notarius derives its default prompt session from the input
|
||||
module and exact input bytes and that ordinary callers should not supply
|
||||
`--session-id`.
|
||||
|
||||
#### Subprocess Invocation
|
||||
|
||||
- Show one concise invocation using `notarius run dnd-session`, explicit
|
||||
absolute `--config`, `--input`, and `--output-dir` paths, and `--json`.
|
||||
- Direct callers to capture stdout and stderr separately, propagate
|
||||
cancellation, impose an operator-appropriate timeout, and wait for process
|
||||
completion before parsing stdout.
|
||||
- State that only exit status zero permits receipt decoding and link to the CLI
|
||||
contract for the complete exit-status definition.
|
||||
- Recommend retaining stderr and the invocation context for diagnosis without
|
||||
logging secrets or transcript content.
|
||||
|
||||
#### Receipt And Bundle Discovery
|
||||
|
||||
- Require callers to accept only supported run-result schema versions while
|
||||
tolerating unknown fields allowed by that version.
|
||||
- Direct callers to obtain the exact run-specific bundle from the receipt's
|
||||
absolute `output_directory`; they must not scan for the newest run directory
|
||||
or construct a run ID.
|
||||
- Require a confinement check when resolving `index_file` beneath the reported
|
||||
bundle root.
|
||||
- Direct callers to discover lane payloads by `lane_id` in `index.json`, then
|
||||
verify descriptor media type and schema identity before decoding them.
|
||||
- Explain that descriptor paths are untrusted relative paths and require the
|
||||
same confinement discipline.
|
||||
|
||||
#### Complete D&D Artifact Inventory
|
||||
|
||||
Include a compact table for the ten lane IDs selected by the maintained
|
||||
complete configuration:
|
||||
|
||||
- `item-registry`;
|
||||
- `npc-registry`;
|
||||
- `location-registry`;
|
||||
- `scene-descriptions`;
|
||||
- `item-occurrences`;
|
||||
- `spells`;
|
||||
- `combat-turns`;
|
||||
- `npc-occurrences`;
|
||||
- `location-occurrences`;
|
||||
- `enemy-events`.
|
||||
|
||||
For each row, give a one-line purpose and link to the corresponding canonical
|
||||
D&D artifact contract. Do not copy its fields or schema rules into the
|
||||
consumer guide.
|
||||
|
||||
Document the four always-published bundle files—`index.json`, `manifest.json`,
|
||||
`rejected.json`, and `warnings.json`—and the complete example's configured
|
||||
`chunk-map.json` and `evidence-context.json` pipeline-wide artifacts. Link to
|
||||
their canonical contracts and distinguish pipeline-wide artifacts from lane
|
||||
outputs.
|
||||
|
||||
The inventory must say that a file is available only when its corresponding
|
||||
artifact was accepted and published. It must not imply that process success
|
||||
guarantees every configured lane.
|
||||
|
||||
#### Downstream Acceptance And Retention
|
||||
|
||||
- Explain that exit status zero can coexist with rejected outputs, warnings,
|
||||
or absent lane descriptors.
|
||||
- Require the consumer to define its required lane set explicitly. Recommend
|
||||
treating all ten lanes as required when the caller claims to consume the
|
||||
complete D&D workflow, while allowing another consumer to adopt a narrower
|
||||
documented policy.
|
||||
- Recommend retaining the receipt, the complete published bundle, and captured
|
||||
diagnostic streams long enough to support provenance and failure analysis.
|
||||
- Explain that `evidence-context.json` is a reading excerpt; authoritative
|
||||
citations remain in lane payloads.
|
||||
- Treat transcripts, lane artifacts, evidence context, manifests, and logs as
|
||||
sensitive campaign data.
|
||||
|
||||
#### Compatibility Checklist
|
||||
|
||||
End with a concise checklist covering process exit, receipt schema, path
|
||||
confinement, pipeline identity, index decoding, required descriptors,
|
||||
descriptor schema/media compatibility, warnings and rejections, checksums or
|
||||
retention, and secure handling. Compatibility should be based on published
|
||||
receipt and artifact contracts rather than parsing a human version string.
|
||||
|
||||
### Update Existing Navigation
|
||||
|
||||
- Add a short link from `docs/consumers/subprocess.md` to the D&D-specific
|
||||
workflow. Keep generic subprocess policy in the existing document.
|
||||
- Add the guide to the documentation links in `README.md`.
|
||||
- Extend the subprocess-consumer row in `docs/development.md` so maintainers
|
||||
working on the D&D workflow are routed to the new guide and the canonical
|
||||
contracts.
|
||||
|
||||
### Verify Canonical Contract Documents
|
||||
|
||||
Review the linked integration documents and the complete example while writing
|
||||
the guide. Correct an integration document only if repository inspection finds
|
||||
an actual stale contract. Do not move schema definitions, field tables, CLI
|
||||
flags, or configuration semantics into the new guide.
|
||||
|
||||
## Narratio Alignment
|
||||
|
||||
The guide may name Narratio as the motivating consumer and identify its current
|
||||
final-trimmed transcript source. It must not claim that Narratio already has a
|
||||
Notarius adapter or extraction stage. Until that feature is implemented,
|
||||
Narratio-specific architecture, configuration, stage behavior, manifest
|
||||
records, and artifact source IDs belong in Narratio's roadmap.
|
||||
|
||||
Once Narratio implements the integration, its own integration documentation
|
||||
should link to this guide and the durable Notarius contracts instead of
|
||||
repeating them.
|
||||
|
||||
## Validation
|
||||
|
||||
Documentation implementation should include:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Also verify all new and changed relative Markdown links, compare the artifact
|
||||
inventory directly with the maintained complete configuration, and confirm
|
||||
that commands and path semantics match the CLI and configuration references.
|
||||
If the repository still has no automated link checker, record that fact and
|
||||
perform a focused manual link review.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A subprocess integrator can follow one D&D-focused guide from a Seriatim
|
||||
transcript through safe discovery of every artifact configured by the
|
||||
complete example.
|
||||
- The guide makes stdout, stderr, exit-status, receipt, and path-confinement
|
||||
responsibilities unambiguous.
|
||||
- The ten configured D&D lanes and both configured pipeline-wide artifacts are
|
||||
listed and linked to their canonical contracts.
|
||||
- The guide distinguishes process success from the caller's required-artifact
|
||||
policy.
|
||||
- The profile-path and reference-path resolution rules are clearly stated.
|
||||
- Existing navigation makes the guide discoverable.
|
||||
- No volatile contract is defined in two places, and no unimplemented Narratio
|
||||
behavior is presented as current.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Implementing or documenting Narratio's future adapter or stage as current
|
||||
Notarius behavior.
|
||||
- Adding a new Notarius command, receipt version, output format, or artifact
|
||||
schema.
|
||||
- Duplicating the complete configuration or individual D&D payload schemas in
|
||||
prose.
|
||||
- Defining a universal partial-result policy for every Notarius consumer.
|
||||
@@ -7,197 +7,16 @@ not as committed release dates.
|
||||
|
||||
## Near-Term Validation And LLM Reliability
|
||||
|
||||
The following work forms one related program but should be promoted into
|
||||
separate feature roadmaps and implemented in dependency order. PromptKit owns
|
||||
structural output repair within one completion. Notarius owns stage candidates,
|
||||
validator chains, semantic rejection policy, and whether another stage attempt
|
||||
is warranted.
|
||||
PromptKit now owns structural output repair within one completion. Notarius
|
||||
owns stage candidates, validator chains, semantic rejection policy, bounded
|
||||
feedback-aware stage retries, validation provenance, and reusable-state
|
||||
eligibility, and the separation of actionable process warnings from quality
|
||||
diagnostics. The remaining near-term work applies those completed foundations
|
||||
to domain review and empirical evaluation.
|
||||
|
||||
### 1. Upgrade To PromptKit v0.8.0
|
||||
|
||||
This item has been promoted to the standalone
|
||||
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md) roadmap. That document owns the
|
||||
release-by-release compatibility review, adopted features, structured-repair
|
||||
policy, target integration boundary, acceptance criteria, and settled design
|
||||
decisions.
|
||||
|
||||
### 2. Feedback-Aware Stage Validation Retries
|
||||
|
||||
- Model Notarius's corrective stage-retry conversation explicitly after
|
||||
PromptKit v0.8.0. The first attempt sends the ordinary complete initial
|
||||
prompt. If application validation rejects the resulting LLM-produced
|
||||
candidate and another stage attempt is available, reconstruct that complete
|
||||
initial prompt byte-for-byte and append exactly two messages: an assistant
|
||||
message containing the defective response and an application-owned user
|
||||
message detailing every applicable semantic validation error and requesting
|
||||
one corrected, complete replacement response. This is a freshly constructed
|
||||
correction request, not continuation of an accumulating conversation.
|
||||
- Use the configured stage `retries` value as the one outer retry budget for
|
||||
this loop. `retries: N` continues to mean at most `N` additional complete
|
||||
chunk, extract, merge, or normalize attempts after the initial attempt,
|
||||
whether an attempt is needed because of a producer error or semantic
|
||||
rejection. Do not add a second semantic-correction count. PromptKit's
|
||||
prompt-level `repair_attempts` budget is independent and internal to each
|
||||
individual LLM completion, and does not consume or replenish the Notarius
|
||||
stage budget.
|
||||
- Extend the framework-managed validation boundary for chunk, extract, merge,
|
||||
and normalize stages so a rejected LLM-produced candidate and its exact raw
|
||||
model response remain available to construct the next stage attempt.
|
||||
Deterministic producers cannot improve by repeating the same inputs; a
|
||||
rejection from a deterministic stage is therefore terminal under the
|
||||
configured rejection policy rather than consuming retries mechanically.
|
||||
- Preserve the original session ID, selected profile, structured-output
|
||||
contract, prompt inputs, and reusable prompt prefix. Carry only the latest
|
||||
candidate and latest aggregate feedback; do not build an unbounded retry
|
||||
conversation. Keep model-facing corrective guidance separate from
|
||||
operator-facing diagnostics, and apply explicit size, redaction, and debug
|
||||
disclosure rules to both.
|
||||
- Run every applicable validator in the configured chain before deciding
|
||||
whether to retry. Do not short-circuit merely because an earlier validator
|
||||
rejected the candidate. Aggregate all semantic rejection reason codes and
|
||||
corrective guidance into the retry message so one retry can address the
|
||||
whole candidate. A validator is applicable only when its declared target and
|
||||
prerequisites can be satisfied; record a deterministic skipped diagnostic
|
||||
rather than invoking a validator on an input it cannot interpret. Initially
|
||||
execute the chain sequentially in configured order so results, diagnostics,
|
||||
costs, and feedback ordering remain deterministic; consider validator
|
||||
concurrency only in response to measured latency.
|
||||
- Continue running independent applicable validators after one validator
|
||||
execution failure so the attempt retains as much useful diagnostic
|
||||
information as practical. Do not present validator operational failures as
|
||||
defects in the producer candidate and do not include them in corrective
|
||||
feedback.
|
||||
- Distinguish three terminal conditions and make their policies configurable
|
||||
at a coherent pipeline or binding scope:
|
||||
- **producer structural failure:** PromptKit could not return a usable
|
||||
structured candidate after its repair budget. Default to `fail_run`; an
|
||||
allowed alternative may record a terminal stage or lane rejection where
|
||||
execution can safely continue, but may not accept the invalid output;
|
||||
- **semantic rejection:** one or more validators completed and rejected the
|
||||
candidate. Default to `fail_run` after corrective stage retries are
|
||||
exhausted; allow an explicit alternative that records the existing
|
||||
rejected-output outcome without advancing that output;
|
||||
- **validator execution failure:** a validator could not produce a valid
|
||||
decision because of generation, structural-output, transport, or internal
|
||||
failure. Default to a genuine warning and an explicitly recorded
|
||||
`validation_incomplete` or equivalent degraded state while allowing the
|
||||
candidate to continue; allow strict configuration to fail the run instead.
|
||||
- An LLM-backed validator uses the same scheduled PromptKit boundary as every
|
||||
other LLM-backed module. Its own response may use PromptKit's bounded
|
||||
structural repair. Distinguish its possible output states:
|
||||
- output rejected by PromptKit's structural contract should consume only the
|
||||
validator prompt's configured PromptKit repair budget;
|
||||
- output that is structurally valid but violates a deterministically
|
||||
checkable validator-result invariant should be classified as a validator
|
||||
execution failure;
|
||||
- output that satisfies the complete validator-result contract is the
|
||||
validator's decision, even though an LLM judgment may remain imperfect.
|
||||
Automatically judging that judgment would require another semantic
|
||||
validator and is outside this feature.
|
||||
If the validator cannot return a contract-valid decision, do not recursively
|
||||
create another Notarius semantic-validation loop around it. Apply the
|
||||
configured validator-failure policy. The default warning must identify the
|
||||
validator and affected stage without exposing sensitive content.
|
||||
- Separate validator execution retry from producer correction. A transient
|
||||
validator operational failure must not automatically discard and regenerate
|
||||
an otherwise usable producer candidate. Any bounded retry of the validator
|
||||
itself should reuse that same immutable candidate and remain subordinate to
|
||||
PromptKit and provider retry behavior.
|
||||
- Preserve attempt-level provenance, cumulative token usage, validator
|
||||
outcomes, aggregated correction feedback, and terminal policy decisions in
|
||||
the debug and manifest models without copying raw source material into
|
||||
ordinary errors or durable summaries.
|
||||
- Define terminal-outcome precedence. A semantic rejection dominates a
|
||||
validator execution failure for the same candidate: use the completed
|
||||
rejections to correct the producer while separately recording incomplete
|
||||
validation. If a later candidate has no semantic rejection but one validator
|
||||
still fails, apply the configured validator-failure policy to that candidate.
|
||||
Never allow a known semantic rejection to become accepted through a
|
||||
warn-and-continue setting, and never accept a structurally invalid producer
|
||||
response. Permissive policy may preserve a rejected-output outcome or accept
|
||||
a structurally valid candidate with explicitly incomplete validation; it may
|
||||
not relabel known-invalid output as approved.
|
||||
|
||||
Before implementation, record the generic validation and retry state machine
|
||||
in an ADR. The ADR should own the separation between PromptKit repair and
|
||||
Notarius correction, use of the existing stage-retry budget, reconstruction of
|
||||
correction conversations, all-applicable-validator aggregation, deterministic
|
||||
validator ordering, non-recursive validator failure handling, outcome
|
||||
precedence, default fail-open/fail-closed choices, configurable terminal
|
||||
policies, and provenance and sensitive-data constraints. A dependency-upgrade
|
||||
ADR is not needed for PromptKit v0.8.0 itself. Current behavior remains
|
||||
authoritative until the validation ADR is implemented and the canonical
|
||||
architecture, configuration, operations, and internal documentation are
|
||||
updated.
|
||||
|
||||
### 3. D&D Combat Scene Semantic Validation
|
||||
|
||||
- Add an optional production LLM-backed D&D validator that determines whether
|
||||
proposed scene boundaries and classifications represent substantive active
|
||||
combat correctly. Its central quality goal is that active combat is kept in
|
||||
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||
- Resolve the validator's exact target before implementation. The current
|
||||
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||
`recap`, and `meta` classification. The preferred initial placement is
|
||||
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||
can compare one proposed kind with the corresponding transcript chunk.
|
||||
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||
question that can be answered from the complete transcript and proposed
|
||||
range map, such as whether one continuous combat was fragmented across
|
||||
inappropriate scene boundaries. Do not duplicate the same classification
|
||||
judgment at both stages. Moving classification into chunk-plan annotations
|
||||
would change the deliberately minimal, annotation-free chunk contract and
|
||||
requires an explicit architecture review before it is selected.
|
||||
- Validate both false negatives and false positives: a non-combat kind must not
|
||||
omit substantive active combat, and a combat kind must be supported by such
|
||||
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||
extraction runs only for an exact `combat` scene classification; semantic
|
||||
review improves the upstream classification but does not replace that gate.
|
||||
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||
structured response schema, and let PromptKit repair structural validator
|
||||
output within its bounded budget. A contract-invalid final validator response
|
||||
is a validator execution failure, not a semantic rejection and not a reason
|
||||
to recursively validate the validator.
|
||||
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||
added calls, latency, and token cost before placing it in the production
|
||||
default chain.
|
||||
- An ADR is not required if classification remains owned by
|
||||
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||
Create or supersede an ADR if the work transfers scene classification into
|
||||
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||
contract.
|
||||
|
||||
### 4. Warning Signal And Presentation Reform
|
||||
|
||||
- Audit every warning producer and representative successful runs. Ordinary
|
||||
success producing dozens of warnings is a failed operator experience: the
|
||||
volume obscures actionable problems and trains operators to ignore the
|
||||
warning channel.
|
||||
- Define a small warning taxonomy that distinguishes actionable degradation,
|
||||
incomplete validation, lossy fallback, and data-quality risk from routine
|
||||
normalization observations or informational diagnostics. Preserve detailed
|
||||
traceability in debug or manifest data without promoting every observation
|
||||
to a top-level CLI warning.
|
||||
- Consider stable deduplication and aggregation by scope and reason code,
|
||||
bounded samples plus omitted counts, and a concise CLI summary with a path to
|
||||
detailed diagnostics. Do not suppress genuine validator execution failures
|
||||
merely to reduce the count.
|
||||
- Decide which warnings affect process status, rejection summaries, durable run
|
||||
receipts, or only debug output. Ensure warning ordering and aggregation are
|
||||
deterministic across concurrent execution.
|
||||
- Establish a representative warning-volume acceptance target and human review
|
||||
workflow before changing individual producers piecemeal. The intended result
|
||||
is not zero warnings; it is a small set in which every surfaced warning merits
|
||||
operator attention.
|
||||
- This work does not require an ADR unless it changes validation acceptance,
|
||||
failure, or durable contract semantics. CLI presentation and diagnostic
|
||||
taxonomy otherwise belong in a feature roadmap followed by updates to their
|
||||
canonical configuration, operations, integration, and internal documents.
|
||||
Near-term reliability work should now be selected from the concrete evaluation
|
||||
and extension opportunities below. The retry, validation, and subprocess
|
||||
foundations described above are implemented current behavior.
|
||||
|
||||
## Near-Term D&D Pipeline
|
||||
|
||||
|
||||
@@ -1,782 +0,0 @@
|
||||
# PromptKit v0.8.0 Upgrade Implementation Plan
|
||||
|
||||
## Purpose
|
||||
|
||||
Implement the target state defined by the
|
||||
[PromptKit v0.8.0 Upgrade](promptkit-v0.8.md): adopt the useful PromptKit
|
||||
v0.6.0, v0.7.0, and v0.8.0 changes; enable one bounded structural correction
|
||||
by default; expose pipeline and binding overrides; preserve safe provider
|
||||
diagnostics; and keep PromptKit behind Notarius's transport-neutral LLM
|
||||
boundary.
|
||||
|
||||
This plan is ordered. Each numbered stage is one implementation prompt for a
|
||||
gpt-5.6-terra coding agent. Complete and validate one stage before beginning
|
||||
the next. Read `docs/development.md` and every policy under `docs/policy/` at
|
||||
the start of each stage, inspect the current code and tests named by that
|
||||
stage, preserve unrelated worktree changes, and update current-behavior
|
||||
documentation in the same stage as the behavior it describes.
|
||||
|
||||
Do not retire this plan or `promptkit-v0.8.md` during implementation. Keep both
|
||||
until the completed work has passed a separate review. Do not implement the
|
||||
future Notarius semantic-validation retry loop, D&D combat-scene validator, or
|
||||
warning redesign as part of this plan.
|
||||
|
||||
## Decisions Fixed For Implementation
|
||||
|
||||
- Pin `gitea.maximumdirect.net/eric/promptkit` v0.8.0 directly, with no
|
||||
`replace`, workspace dependency, or vendored source.
|
||||
- Every maintained eligible production prompt defaults to exactly one
|
||||
PromptKit structural repair attempt.
|
||||
- Add the exact configuration key
|
||||
`structured_output_repair_attempts` at pipeline scope and on LLM-backed
|
||||
module and validator bindings.
|
||||
- Effective precedence is binding value, then pipeline value, then the prompt's
|
||||
declared `repair_attempts` value. Omission inherits; explicit zero disables
|
||||
structural repair at that scope.
|
||||
- Accepted values are integers from zero through three. Explicit null and
|
||||
non-integer values are invalid. An explicit binding value on a deterministic
|
||||
module or validator is invalid. A pipeline value is applied only to selected
|
||||
LLM-backed bindings and does not make deterministic bindings invalid.
|
||||
- Keep file configuration version 4. This is an additive pre-release field and
|
||||
does not require parallel versioned behavior.
|
||||
- Use `StructuredOutputRepairAttempts *int` for presence-aware internal Go
|
||||
fields. Clone pointers at every ownership boundary.
|
||||
- A configured override never replaces schema identity, output format, or
|
||||
validation mode. The PromptKit adapter calls `InspectPrompt`, copies the
|
||||
complete normalized prompt-owned output contract, changes only
|
||||
`RepairAttempts`, and supplies the complete replacement on `RunRequest`.
|
||||
Do not add an inspection cache initially.
|
||||
- PromptKit repair is internal to one `CompleteStructured` call and does not
|
||||
consume or replenish a binding's existing `retries` budget.
|
||||
- Add `RepairAttempts int` to Notarius's structured-completion response. It is
|
||||
the actual corrective-call count reported by PromptKit; token usage remains
|
||||
PromptKit's cumulative usage and must not be summed again.
|
||||
- A valid repaired response is successful and produces no warning solely
|
||||
because repair occurred. Exhausted structural validation maps to
|
||||
`ErrInvalidStructuredOutput` with the final candidate and debug material
|
||||
retained.
|
||||
- Add an application-owned generation-error sentinel and typed status-bearing
|
||||
error. PromptKit error types must not cross `internal/framework/llm`.
|
||||
- HTTP status may appear in the application-owned generation error. Provider
|
||||
code, type, and message are excluded from ordinary errors, warnings,
|
||||
manifests, cache, and checkpoint identity; they may appear only in an
|
||||
explicitly requested debug trace after Notarius redaction.
|
||||
- Profile inheritance is owned entirely by PromptKit. Notarius passes sources
|
||||
through, inspects and records the resolved target, and does not parse or merge
|
||||
`base_profile` itself.
|
||||
- PromptKit's built-in `rakestrawhome` backend and
|
||||
`rakestrawhome-gemma-4-31b` profile are available generically. Notarius does
|
||||
not register, shadow, or select them by default.
|
||||
- Missing optional credential environment values are allowed to reach the
|
||||
provider without `Authorization`; Notarius does not recreate v0.5.0's local
|
||||
failure or add provider-specific authentication logic.
|
||||
- No dependency-upgrade ADR is required. Update architecture only with the
|
||||
durable ownership distinction between PromptKit structural repair and
|
||||
Notarius stage/semantic validation policy.
|
||||
|
||||
## Stage 1: Upgrade The Dependency And Establish A Clean v0.8.0 Baseline ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Move the repository to PromptKit v0.8.0, resolve source-compatibility issues,
|
||||
and establish a passing baseline before adopting new behavior.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Re-read the upstream v0.6.0, v0.7.0, and v0.8.0 release guides and the
|
||||
v0.8.0 package consumer and format documentation. Treat the pinned v0.8.0
|
||||
tag, not the sibling checkout's moving branch, as authoritative.
|
||||
2. Update `go.mod` and `go.sum` to PromptKit v0.8.0 and run `go mod tidy` with
|
||||
`GOWORK=off`.
|
||||
3. Compile before making compatibility edits. Correct only actual source or
|
||||
behavior incompatibilities. In particular:
|
||||
- convert any positional `promptkit.Profile` or
|
||||
`promptkit.OpenAICompatibleProfileConfig` literals to keyed literals;
|
||||
- confirm Notarius does not register the newly reserved `rakestrawhome`
|
||||
backend ID; and
|
||||
- preserve `PrepareExecution`/`Details`/`RunPrepared` snapshot ownership,
|
||||
`Discard`, session forwarding, reasoning override, profile preflight,
|
||||
and capacity adaptation.
|
||||
4. Change `promptKitBuiltinProfileCatalogID` in
|
||||
`internal/framework/llm/promptkit_profile_fingerprint.go` from the v0.5.0
|
||||
catalog marker to an opaque v0.8.0 marker. Do not hash PromptKit internal
|
||||
files or include catalog content in manifests.
|
||||
5. Update `docs/integrations/pkg-promptkit.md` to pin and link v0.8.0 and to
|
||||
state that this stage still leaves the production prompt-declared repair
|
||||
budget at its current value. Do not document later configuration or default
|
||||
behavior before it exists.
|
||||
6. Update only those existing tests whose public PromptKit types or stable
|
||||
v0.8.0 behavior genuinely changed. Do not rewrite tests merely to match
|
||||
upstream diagnostic wording.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go mod tidy -diff
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./cmd/notarius
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- `go list -m gitea.maximumdirect.net/eric/promptkit` reports v0.8.0.
|
||||
- There is no PromptKit `replace`, active Go workspace dependency, or vendor
|
||||
tree.
|
||||
- The adapter still uses one frozen prepared execution and all existing LLM
|
||||
tests pass.
|
||||
- Checkpoint profile identity includes the v0.8.0 built-in catalog marker.
|
||||
- Current integration documentation pins v0.8.0 without claiming that
|
||||
later stages are already active.
|
||||
- The full ordinary test suite, vet, and command build pass.
|
||||
|
||||
## Stage 2: Verify v0.6.0 Compatibility And Hardening ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Audit Notarius's assets and boundary values against PromptKit v0.6.0's stricter
|
||||
source, path, endpoint, JSON, and cancellation contracts, fixing only concrete
|
||||
incompatibilities.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inspect `internal/framework/llm/asset_registry.go`, prompt/profile source
|
||||
composition, all registered asset roots, the conventional local backend,
|
||||
and their focused tests.
|
||||
2. Exercise every production asset registry through PromptKit engine
|
||||
construction and the existing production composition tests. Confirm that:
|
||||
- YAML IDs and versions, not filenames, select definitions;
|
||||
- every `content_file` path is exact, relative, contained, and points to a
|
||||
regular embedded file;
|
||||
- every schema and JSON asset is one complete JSON value;
|
||||
- every current output contract is valid under v0.8.0; and
|
||||
- unrelated malformed definitions do not create a second Notarius identity
|
||||
or fallback mechanism.
|
||||
3. Review local endpoint parsing and validation. Retain a narrower Notarius
|
||||
rule only if it has independent application value; otherwise rely on
|
||||
PromptKit's absolute HTTP/HTTPS URL contract. Never accept a value that the
|
||||
adapter will later reject.
|
||||
4. Review conversion of Notarius variables, inputs, profile extras, and debug
|
||||
values at the adapter boundary for PromptKit's bounded JSON-compatible-value
|
||||
rules. Do not add a second generic JSON walker or duplicate upstream numeric
|
||||
limits.
|
||||
5. Verify cancellation and deadline identity through existing adapter tests.
|
||||
Add or refine one focused regression only if Notarius currently destroys an
|
||||
`errors.Is`-relevant context or transport error that the application owns.
|
||||
6. Do not add a cross-operation schema cache, artifact cache, provider-body
|
||||
reader, or duplicate JSON framing validation; v0.6.0 owns those mechanisms.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run the focused asset, profile-source, and adapter packages, then the ordinary
|
||||
and race-enabled suites:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Tests must remain offline and should validate Notarius's assembled boundary,
|
||||
not reproduce PromptKit's internal path, JSON-depth, or response-size matrices.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every maintained embedded prompt, schema, and fallback profile can be loaded
|
||||
through the assembled v0.8.0 engine.
|
||||
- Current local endpoint and JSON-compatible values either satisfy the stricter
|
||||
upstream contract or fail during preparation with safe diagnostics.
|
||||
- No duplicate PromptKit-owned cache, JSON, or response-bound mechanism is
|
||||
introduced.
|
||||
- Cancellation and deadline behavior remains discoverable at the Notarius
|
||||
boundary.
|
||||
- Ordinary and race-enabled tests pass.
|
||||
|
||||
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Make the useful PromptKit v0.7.0 profile and backend behavior work through
|
||||
Notarius's existing generic profile boundary without adding provider-specific
|
||||
composition logic.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inspect `promptkit_profiles.go`, `asset_registry.go`, profile fingerprinting,
|
||||
CLI profile preflight, profile provenance recording, and their tests before
|
||||
editing.
|
||||
2. Add an offline integration test using a temporary operator profile source
|
||||
whose leaf uses `base_profile`. Prove that:
|
||||
- preflight reports the leaf ID;
|
||||
- the effective backend, model, reasoning, and other inherited values match
|
||||
the resolved PromptKit target;
|
||||
- execution uses the same resolved target as inspection; and
|
||||
- a missing parent or cycle fails before provider generation with a safe
|
||||
profile-load diagnostic.
|
||||
Do not duplicate PromptKit's entire field-by-field merge test matrix.
|
||||
3. Add a checkpoint-safety test showing that changing a parent definition in
|
||||
an operator profile directory changes Notarius's profile-source fingerprint
|
||||
while profile content and paths remain absent from the fingerprint value.
|
||||
Retain the v0.8.0 catalog marker as coverage for built-in-parent changes.
|
||||
4. Verify `rakestrawhome-gemma-4-31b` through the ordinary profile inspector.
|
||||
Assert its selected backend reaches Notarius's application-owned inspection
|
||||
and provenance fields. Use a fake PromptKit client or transport if execution
|
||||
coverage is needed; never contact the live service or require credentials.
|
||||
5. Verify that Notarius registers no `rakestrawhome` override and that the
|
||||
existing `local` registration remains independent.
|
||||
6. Add one `httptest`-backed adapter integration test for a filesystem profile
|
||||
with a missing optional `api_key_env`. The request must reach the test server
|
||||
without an `Authorization` header. Add a focused in-memory PromptKit profile
|
||||
test for `APIKeyRequired` only if needed to prove Notarius preserves upstream
|
||||
preflight behavior; do not expose a new operator profile API.
|
||||
7. Keep `assets/dnd/profiles/dnd-extraction.yaml` standalone and unchanged. No
|
||||
matching v0.8.0 built-in profile owns its `openai/gpt-5.6-luna` target.
|
||||
8. Update the current profile-source, deployment, and pinned-integration
|
||||
sections in `docs/config.md`, `docs/operations.md`,
|
||||
`docs/internal/llm.md`, and `docs/integrations/pkg-promptkit.md`. Link to the
|
||||
pinned PromptKit format rules for inheritance. Explain that filesystem
|
||||
profiles cannot express PromptKit's in-memory `APIKeyRequired` field and
|
||||
that an optional missing credential may result in a provider 401/403.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/core/config ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/cli
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Inherited operator profiles resolve identically during preflight and
|
||||
execution, with the leaf ID and effective target kept distinct.
|
||||
- Parent changes invalidate checkpoint reuse without leaking profile content or
|
||||
paths.
|
||||
- Rakestrawhome is available through generic PromptKit profile handling and is
|
||||
not selected by default or registered by Notarius.
|
||||
- Missing optional credentials omit authorization and reach the controlled
|
||||
test provider; explicitly required credentials retain upstream behavior.
|
||||
- Current documentation accurately describes the implemented profile and
|
||||
credential behavior without duplicating PromptKit's merge algorithm.
|
||||
|
||||
## Stage 4: Adapt Structured Generation Errors Safely ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Use PromptKit v0.7.0's structured generation errors for stable status
|
||||
classification and debug-only provider diagnostics without leaking PromptKit
|
||||
types or sensitive provider text.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. In `internal/framework/contracts`, add:
|
||||
- `ErrLLMGeneration` as the provider-neutral generation-failure sentinel;
|
||||
- an application-owned `LLMGenerationError` with private status and safe
|
||||
diagnostic fields, `Error`, `Unwrap`, and `StatusCode` methods; and
|
||||
- a constructor that accepts a nonnegative status and an already-redacted
|
||||
diagnostic. Status zero means no HTTP status was available.
|
||||
Ordinary callers may inspect status with `errors.As` and category with
|
||||
`errors.Is`, but cannot obtain provider code, type, or message from the
|
||||
error.
|
||||
2. Add an application-owned `LLMDebugProviderError` with `status_code`, `code`,
|
||||
`type`, and `message` fields, referenced optionally from
|
||||
`LLMDebugResponse`. This is debug material, not a manifest or durable public
|
||||
artifact contract.
|
||||
3. In `PromptKitClient.CompleteStructured`, preserve precedence in this order:
|
||||
caller context cancellation/deadline, PromptKit capacity error, structured
|
||||
PromptKit generation error, then other PromptKit generation failures.
|
||||
Map every generation failure to `ErrLLMGeneration`; map
|
||||
`*promptkit.GenerationError` to `LLMGenerationError` with its status.
|
||||
Never wrap or return the PromptKit error value itself.
|
||||
4. Keep the ordinary diagnostic limited to PromptKit's safe default error
|
||||
formatting after bearer and known-credential redaction. Do not append
|
||||
`ProviderCode`, `ProviderType`, or `ProviderMessage` to it.
|
||||
5. For an explicitly requested debug path, preserve prepared prompt details and
|
||||
attach the PromptKit provider code, type, and message after:
|
||||
- reading only the selected prepared target's `APIKeyEnv`, if any, to obtain
|
||||
the exact known credential solely for redaction;
|
||||
- applying `RedactSecrets` and the existing bearer/key-pattern redaction;
|
||||
- retaining PromptKit's already-normalized bounds; and
|
||||
- discarding the credential value immediately rather than storing it.
|
||||
Do not scan unrelated environment variables.
|
||||
6. Return prompt/debug material alongside the error so the existing debug LLM
|
||||
wrapper can persist it only when debug recording is enabled. Confirm that
|
||||
provider fields do not appear in ordinary error text, warnings, manifests,
|
||||
cache, checkpoint data, or a run without debug output.
|
||||
7. Refactor error mapping into small helpers if needed to keep
|
||||
`CompleteStructured` readable; do not create provider-specific policy in
|
||||
modules or the pipeline runner.
|
||||
8. Update the error and observability sections of `docs/internal/llm.md` and
|
||||
`docs/integrations/pkg-promptkit.md`. Keep operator disclosure rules in
|
||||
`docs/operations.md` concise and link to the internal boundary where useful.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
- Use `httptest.Server` to return representative structured 400 and 503
|
||||
responses. Assert `errors.Is(ErrLLMGeneration)`, `errors.As` to the
|
||||
application-owned type, and the exact status without asserting complete
|
||||
human wording.
|
||||
- Include a provider message containing the selected test credential and a
|
||||
bearer-shaped value. Verify both are absent from the ordinary error and
|
||||
debug artifact, while a non-sensitive marker appears only in the requested
|
||||
debug trace.
|
||||
- Retain existing capacity and context tests to prove their more specific
|
||||
classifications still win.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline ./internal/cli
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- PromptKit generation errors never escape the adapter error chain.
|
||||
- All generation failures match `ErrLLMGeneration`; structured non-success
|
||||
responses expose only application-owned HTTP status to ordinary callers.
|
||||
- Provider code, type, and message are available only in an explicitly
|
||||
requested, redacted debug trace.
|
||||
- Capacity and context classifications remain unchanged and more specific.
|
||||
- Security tests prove selected credentials and bearer tokens are not leaked.
|
||||
|
||||
## Stage 5: Add Adapter-Level Structured Repair Support ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Teach the transport-neutral completion boundary and PromptKit adapter to apply
|
||||
an optional repair override and report actual repair behavior, without yet
|
||||
exposing the setting in pipeline configuration.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` to
|
||||
`contracts.StructuredCompletionRequest`. Copy the pointed-to value wherever
|
||||
requests are cloned or retained.
|
||||
2. Add `RepairAttempts int` to `contracts.StructuredCompletionResponse`. It is
|
||||
the actual number of corrective generation calls, not the configured budget
|
||||
and not the number of total candidates.
|
||||
3. Validate a non-nil request value as zero through three at the adapter
|
||||
boundary so programmatic callers cannot bypass later file/config validation.
|
||||
4. When the request value is nil, leave `promptkit.RunRequest.Validation` nil
|
||||
so the prompt's complete contract remains authoritative.
|
||||
5. When the value is non-nil:
|
||||
- call `Engine.InspectPrompt(ctx, promptID, promptVersion)`;
|
||||
- copy `PromptInspection.OutputContract` by value;
|
||||
- replace only `RepairAttempts`;
|
||||
- pass the complete copied contract as `RunRequest.Validation`; and
|
||||
- prepare and execute exactly as before.
|
||||
Do not infer or hard-code schema paths, validation modes, or formats. Do not
|
||||
cache inspection in this stage.
|
||||
6. Map `result.Validation.RepairAttempts` to the response and leave
|
||||
`result.Usage` cumulative values unchanged. The existing debug validation
|
||||
object and prepared output contract should show actual and configured values
|
||||
respectively.
|
||||
7. Preserve result semantics:
|
||||
- valid initial and repaired candidates decode normally;
|
||||
- repair exhaustion returns the final raw candidate/debug material with an
|
||||
error matching `ErrInvalidStructuredOutput`;
|
||||
- explicit empty or whitespace-only content follows PromptKit validation;
|
||||
- missing/null/non-string content remains a generation/provider failure;
|
||||
- corrective-call generation errors use Stage 4's application-owned mapping;
|
||||
and
|
||||
- context cancellation wins at every error boundary.
|
||||
8. Keep `CompleteStructured` and its helpers provider neutral outside this
|
||||
adapter package. Do not expose PromptKit validation or inspection types.
|
||||
9. Update only the adapter-owned repair behavior in `docs/internal/llm.md` and
|
||||
`docs/integrations/pkg-promptkit.md`. State that public pipeline configuration
|
||||
and the production default are added by later stages of this plan.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Add adapter-level behavioral tests using a deterministic fake PromptKit LLM:
|
||||
|
||||
- nil override uses the prompt declaration;
|
||||
- explicit zero overrides a positive prompt declaration without dropping its
|
||||
JSON Schema contract;
|
||||
- explicit one turns an invalid first candidate followed by a valid candidate
|
||||
into one successful response with the final raw bytes, actual repair count
|
||||
one, and cumulative usage;
|
||||
- repair exhaustion returns the final candidate and validation diagnostics as
|
||||
`ErrInvalidStructuredOutput`;
|
||||
- explicit empty content is eligible for repair;
|
||||
- a corrective generation failure maps through Stage 4; and
|
||||
- invalid direct values below zero or above three fail before provider work.
|
||||
|
||||
Do not assert PromptKit's exact assistant/user correction prose or copy its
|
||||
full internal repair matrix.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline
|
||||
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The adapter changes only repair count when applying a request override.
|
||||
- Nil and explicit zero remain distinct.
|
||||
- Repaired success returns final raw output, cumulative usage, and actual count
|
||||
without a warning.
|
||||
- Exhaustion, empty content, corrective generation failure, and cancellation
|
||||
match the target semantics.
|
||||
- No PromptKit type crosses the LLM package boundary.
|
||||
|
||||
## Stage 6: Propagate Repair Policy Through Framework Requests ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Carry an optional effective repair budget from each resolved stage or validator
|
||||
binding to its module request without changing public file configuration yet.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` alongside `LLMProfile` to every
|
||||
stage request that can belong to an LLM-backed binding:
|
||||
- `ParseRequest`;
|
||||
- `ChunkRequest`;
|
||||
- `TypedExtractionRequest`;
|
||||
- `TypedMergeRequest`;
|
||||
- `TypedNormalizeRequest`;
|
||||
- `OutputRequest`;
|
||||
- `TypedValidationRequest`;
|
||||
- `ChunkValidationRequest`; and
|
||||
- `SerializedValidationRequest`.
|
||||
2. Add the same optional field to the erased/internal request carriers used by
|
||||
registry builders, preparation, runner stage attempts, validator targets,
|
||||
retry closures, and debug wrappers. Copy pointer values; never share a
|
||||
mutable pointer owned by configuration.
|
||||
3. At every runner stage invocation, obtain the value from the exact resolved
|
||||
producer binding. At every validator invocation, obtain it from that exact
|
||||
resolved validator binding. Do not use the producer's value for a validator
|
||||
or vice versa.
|
||||
4. Ensure all retry attempts for the same binding receive the same effective
|
||||
structural-repair value. Do not decrement it in Notarius; PromptKit owns the
|
||||
inner budget independently on each `CompleteStructured` call.
|
||||
5. Extend `semanticreconcile.Request` with the optional field and carry it into
|
||||
each generic reconciliation completion. A batched reconciliation may make
|
||||
several completion calls; each call receives the same effective budget.
|
||||
6. Update registry erasure/adaptation code for typed merge, normalize, and
|
||||
validation requests so no field is lost. Preserve input/output support even
|
||||
though current production input and output modules are deterministic.
|
||||
7. Add focused framework tests for one chunk producer, one extraction
|
||||
producer, one normalizer, and one LLM-backed validator. Verify exact pointer
|
||||
value propagation and separation between producer and validator settings.
|
||||
Do not add repetitive tests for every generic adapter.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/contracts ./internal/framework/pipeline ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test -race ./internal/framework/pipeline ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every stage and validator request can carry a detached optional repair value.
|
||||
- The runner sources the value from the exact resolved binding.
|
||||
- Producer and validator values cannot overwrite one another.
|
||||
- Stage retries reuse but do not mutate or consume the inner repair budget.
|
||||
- Semantic reconciliation forwards the budget to every one of its completion
|
||||
calls.
|
||||
- Existing behavior remains unchanged while all values are nil.
|
||||
|
||||
## Stage 7: Forward Repair Policy From Every LLM-Backed Module ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Complete the internal end-to-end path by having every production LLM-backed
|
||||
module forward its stage request value to `CompleteStructured`.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Inventory every production `CompleteStructured` call with code search before
|
||||
editing. The expected current owners include:
|
||||
- `dnd/scenes` chunking;
|
||||
- the combat-turn, enemy-event, item-occurrence, item-registry,
|
||||
location-occurrence, location-registry, NPC-occurrence, NPC-registry,
|
||||
scene-description, and spell extractors; and
|
||||
- generic semantic reconciliation used by the item, location, and NPC
|
||||
registry normalizers.
|
||||
Reconcile this list with the actual repository; do not omit a newly added
|
||||
production caller merely because it is not named here.
|
||||
2. In each direct caller, set
|
||||
`StructuredCompletionRequest.StructuredOutputRepairAttempts` from the
|
||||
corresponding stage request. Clone the pointer or use a small shared helper
|
||||
if that reduces repeated ownership mistakes without moving domain logic.
|
||||
3. Ensure D&D registry normalizers pass their typed normalize request value into
|
||||
`semanticreconcile.Request`, and that the generic engine forwards it as
|
||||
established in Stage 6.
|
||||
4. Update existing module prompt-mapping tests that already inspect a captured
|
||||
structured-completion request to assert the new field. Do not create a new
|
||||
one-test-per-module suite solely to memorialize field plumbing; rely on the
|
||||
existing request-contract tests plus a final complete call-site audit.
|
||||
5. Search again after editing for production `CompleteStructured` calls and
|
||||
verify each either forwards the field or documents why it cannot receive a
|
||||
pipeline binding. Test-only fakes need only preserve the field when their
|
||||
contract test depends on it.
|
||||
6. Do not set a module-specific fallback value. Nil must reach the adapter so
|
||||
the prompt declaration remains authoritative.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run focused D&D and semantic-reconciliation packages, then the full suite:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/modules/dnd/... ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test -race ./internal/modules/dnd/... ./internal/framework/semanticreconcile
|
||||
GOWORK=off go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every production LLM-backed completion receives the exact stage or validator
|
||||
repair value.
|
||||
- No module invents a default or imports PromptKit.
|
||||
- Registry normalizers preserve the value through semantic reconciliation.
|
||||
- Existing request-contract tests remain concise and pass.
|
||||
- A final call-site audit finds no silent production omission.
|
||||
|
||||
## Stage 8: Add The Public Repair Configuration Contract ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Add presence-aware pipeline and binding configuration for
|
||||
`structured_output_repair_attempts` without yet changing runtime resolution.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `StructuredOutputRepairAttempts *int` to
|
||||
`pipeline.PipelineProfile` and `pipeline.ModuleBinding`, using
|
||||
`json:"structured_output_repair_attempts,omitempty"`.
|
||||
2. Add presence-aware YAML support at pipeline and object-binding scope:
|
||||
- exact key `structured_output_repair_attempts`;
|
||||
- integer values zero through three;
|
||||
- explicit null, non-integer, and out-of-range values rejected with scoped
|
||||
diagnostics; and
|
||||
- scalar shorthand bindings continue to omit the binding override.
|
||||
Preserve file configuration version 4.
|
||||
3. Update every configuration clone, conversion, redaction, summary, and JSON
|
||||
round-trip carrier. Copy pointers by value into newly allocated storage so
|
||||
parsed, configured, and redacted values do not alias.
|
||||
4. Preserve omission versus explicit zero through YAML parsing, profile
|
||||
inheritance, module-binding object form, and JSON round trips. Keep scalar
|
||||
shorthand bindings equivalent to omission.
|
||||
5. Do not add a top-level `promptkit.repair_attempts` setting or CLI override.
|
||||
6. Add concise parser and ownership tests. Defer execution-class checks,
|
||||
effective precedence, resolved digests, and runtime forwarding to Stage 9,
|
||||
where module metadata is available.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
At the parser/config boundary, test omitted, explicit zero, positive bounds,
|
||||
negative, above-three, null, non-integer, scalar shorthand, cloning, redaction,
|
||||
and JSON round-trip behavior. Use relational boundary tests for the allowed
|
||||
range and avoid duplicating the same cases at every layer.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/core/config ./internal/cli
|
||||
GOWORK=off go test -race ./internal/core/config
|
||||
GOWORK=off go test ./...
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The exact public field parses at pipeline and object-binding scope with the
|
||||
fixed range.
|
||||
- Nil and explicit zero remain distinguishable through parsing, cloning,
|
||||
inheritance, redaction, summaries, and round trips.
|
||||
- Both maintained configurations remain valid without requiring the new field.
|
||||
- No runtime or prompt default has changed prematurely.
|
||||
|
||||
## Stage 9: Resolve And Apply Repair Configuration ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Resolve the public field against module execution classes, incorporate the
|
||||
effective value into pipeline identity, and connect it to the request plumbing
|
||||
completed in Stages 6 and 7.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. During resolution, compute the effective value for every selected binding:
|
||||
- explicit binding value wins;
|
||||
- otherwise an explicit pipeline value applies to an LLM-backed binding;
|
||||
- otherwise leave nil for prompt-owned policy.
|
||||
Apply the pipeline value to LLM-backed validators as well as producers.
|
||||
2. Reject an explicit binding value on a deterministic module or deterministic
|
||||
validator using the same execution-class knowledge used for `llm_profile`.
|
||||
Do not reject a pipeline-level value merely because a selected pipeline also
|
||||
contains deterministic bindings; simply do not apply it to those bindings.
|
||||
3. Clone every resolved pointer so the parsed pipeline, resolved profile,
|
||||
redacted summaries, and runner requests have distinct ownership.
|
||||
4. Include the effective field in resolved pipeline JSON and digest input. A
|
||||
change between nil, zero, and a positive value must change the resolved
|
||||
digest when it changes an LLM-backed selected binding. Unselected lanes must
|
||||
retain the repository's existing digest and selection semantics.
|
||||
5. Pass the resolved value into the Stage 6 request field for every selected
|
||||
input, chunk, extract, merge, normalize, output, and validator binding.
|
||||
6. Update `docs/config.md` as the canonical field, range, and precedence
|
||||
contract; `docs/internal/pipeline.md` as the resolution owner; and
|
||||
`docs/operations.md` for the distinction from binding `retries`. The
|
||||
prompt-owned production default remains unchanged until Stage 10.
|
||||
7. Add focused resolution and runner tests. Cover representative execution
|
||||
classes rather than repeating the same assertion for every module type.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Test:
|
||||
|
||||
- binding over pipeline over nil precedence;
|
||||
- inheritance into each selected LLM-backed stage and validator;
|
||||
- no inheritance into deterministic bindings;
|
||||
- explicit deterministic-binding rejection;
|
||||
- detached pointers;
|
||||
- runner forwarding for representative producer and validator bindings; and
|
||||
- digest changes for execution-relevant nil, zero, and positive changes.
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/pipeline ./internal/cli
|
||||
GOWORK=off go test -race ./internal/framework/pipeline
|
||||
GOWORK=off go test ./...
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- The exact public field has the fixed binding-over-pipeline-over-prompt
|
||||
precedence for every selected LLM-backed producer and validator.
|
||||
- Deterministic binding misuse fails during resolution before execution, while
|
||||
a pipeline value coexists with deterministic bindings.
|
||||
- Nil and explicit zero remain distinguishable through resolution, runtime,
|
||||
summaries, and digests.
|
||||
- A policy change invalidates checkpoint identity when it changes an effective
|
||||
selected binding.
|
||||
- Current configuration, pipeline, and operations documentation matches the
|
||||
implemented behavior.
|
||||
|
||||
## Stage 10: Enable The Default, Finish Documentation, And Verify The Feature ✅
|
||||
|
||||
### Goal
|
||||
|
||||
Set the accepted production default of one repair, reconcile all canonical
|
||||
documentation, and run the full repository verification pass.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Change `repair_attempts: 0` to `repair_attempts: 1` in every maintained
|
||||
production prompt manifest that produces structured output, including the
|
||||
generic semantic-reconciliation prompt and every D&D chunk, extraction, and
|
||||
registry-normalization prompt. Do not mechanically change unrelated test
|
||||
fixtures whose purpose is to exercise zero.
|
||||
2. Inspect every production prompt output contract after the edit. Confirm that
|
||||
each positive budget uses `basic`, `json`, or `json_schema`, remains no
|
||||
greater than three, and retains its existing format and schema path.
|
||||
3. Add or refine the smallest durable assembled-assets test that proves the
|
||||
production engine can prepare the maintained prompts with the activated
|
||||
contracts. Do not add a brittle test that asserts an exact prompt count,
|
||||
file count, message prose, correction text, or asset length. The public
|
||||
default may be tested at one canonical assembled boundary because its
|
||||
literal value is an operational contract.
|
||||
4. Confirm a successful repair does not create a warning and that exhausted
|
||||
repair remains `ErrInvalidStructuredOutput`. Verify the debug prompt records
|
||||
the configured contract, the debug response records actual repair count,
|
||||
and cumulative usage is not double-counted.
|
||||
5. Confirm scheduling behavior with one focused test or existing coverage: the
|
||||
Notarius scheduled client admits one logical `CompleteStructured` operation
|
||||
while PromptKit may make serial corrective provider calls inside it. Do not
|
||||
attempt to reacquire a Notarius permit from inside PromptKit or add a second
|
||||
scheduler.
|
||||
6. Reconcile current-state documentation:
|
||||
- `docs/integrations/pkg-promptkit.md` owns the pinned upstream boundary;
|
||||
- `docs/config.md` owns field names, range, default, and precedence;
|
||||
- `docs/operations.md` owns latency/cost, optional credentials, concurrency,
|
||||
timeout, and the upper-bound formula;
|
||||
- `docs/internal/llm.md` owns inspection-based contract replacement,
|
||||
cumulative usage, actual repair count, generation errors, and debug data;
|
||||
- `docs/internal/pipeline.md` owns effective policy propagation and the
|
||||
separation from stage retries; and
|
||||
- `docs/policy/architecture.md` adds only the durable rule that PromptKit
|
||||
owns deterministic structural repair inside one completion while Notarius
|
||||
owns stage attempts and semantic validation.
|
||||
7. Remove current-behavior claims that PromptKit is v0.5.0, that every
|
||||
production repair budget is zero, or that PromptKit is always single-pass.
|
||||
Do not alter historical release notes or archived roadmaps.
|
||||
8. Keep the maintained minimal and complete examples secret-free and valid.
|
||||
They may omit the new field to demonstrate the default; do not add a
|
||||
redundant complete profile or Rakestrawhome example merely to exercise an
|
||||
upstream catalog entry.
|
||||
9. Review `docs/roadmap/future.md` only for consistency. Leave the future
|
||||
feedback-aware stage retry, combat-scene validator, and warning-reform work
|
||||
unimplemented and clearly separate.
|
||||
|
||||
### Tests And Validation
|
||||
|
||||
Run focused tests first, then all repository checks:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./internal/framework/llm ./internal/framework/pipeline ./internal/framework/semanticreconcile ./internal/modules/dnd/...
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./cmd/notarius
|
||||
GOWORK=off go mod tidy -diff
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Also perform focused repository searches that exclude `docs/roadmap/archive/`
|
||||
and historical release notes:
|
||||
|
||||
- no active v0.5.0 PromptKit pins or links remain;
|
||||
- no maintained production prompt still declares `repair_attempts: 0`;
|
||||
- every production `CompleteStructured` caller forwards the repair field; and
|
||||
- no provider code, type, or message is added to ordinary errors, warnings,
|
||||
manifests, cache, or checkpoint schemas.
|
||||
|
||||
If the repository's source-release checker is available and the ordinary
|
||||
checks above pass, run `./scripts/check-release-source.sh v0.0.0` as the final
|
||||
integrated validation. It must not create a tag, release note, or repository
|
||||
artifact.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Every maintained structured prompt defaults to one corrective call and can
|
||||
be overridden to zero through three at pipeline or binding scope.
|
||||
- A real assembled Notarius completion follows the PromptKit v0.8.0 repair
|
||||
contract without changing prompt schema identity or cacheable prefix.
|
||||
- Actual repair count, cumulative usage, error classification, debug-only
|
||||
provider diagnostics, scheduling, and checkpoint identity match the feature
|
||||
roadmap.
|
||||
- Profile inheritance, Rakestrawhome availability, optional credentials, and
|
||||
v0.6.0 hardening remain covered and documented.
|
||||
- All canonical documentation describes implemented v0.8.0 behavior in its
|
||||
assigned home and leaves future semantic validation work in the roadmap.
|
||||
- Maintained examples validate, all ordinary/race/vet/build/module checks pass,
|
||||
and the worktree contains no generated or sensitive artifacts.
|
||||
@@ -1,520 +0,0 @@
|
||||
# PromptKit v0.8.0 Upgrade
|
||||
|
||||
## Status
|
||||
|
||||
Proposed.
|
||||
|
||||
## Purpose
|
||||
|
||||
Upgrade Notarius from PromptKit v0.5.0 to v0.8.0 and deliberately adopt the
|
||||
useful correctness, profile-composition, provider-diagnostic, backend, and
|
||||
structured-output-repair capabilities introduced in PromptKit v0.6.0, v0.7.0,
|
||||
and v0.8.0.
|
||||
|
||||
The upgrade should improve structured-output reliability without confusing
|
||||
PromptKit's bounded deterministic repair with Notarius's existing stage retry
|
||||
budget or the future feedback-aware semantic-validation loop. PromptKit types
|
||||
and provider behavior must remain behind Notarius's transport-neutral LLM
|
||||
boundary.
|
||||
|
||||
## Current State
|
||||
|
||||
Notarius currently pins PromptKit v0.5.0. Its production adapter prepares one
|
||||
frozen execution, records credential-redacted details, and runs that same
|
||||
prepared value. It maps PromptKit capacity failures to an application-owned
|
||||
error, maps failed structured validation to `ErrInvalidStructuredOutput`, and
|
||||
returns PromptKit's raw validated bytes and usage metadata.
|
||||
|
||||
Every maintained production prompt uses JSON Schema validation and currently
|
||||
declares `repair_attempts: 0`. Notarius stage bindings separately expose
|
||||
`retries`, which reruns a complete stage operation after an error or rejected
|
||||
candidate. The two mechanisms have different ownership and must remain
|
||||
independent.
|
||||
|
||||
Notarius also maintains:
|
||||
|
||||
- embedded prompt, schema, and fallback-profile filesystems;
|
||||
- operator profile-file and profile-directory sources;
|
||||
- one optional conventional `local` backend registration;
|
||||
- explicit profile preflight through PromptKit inspection;
|
||||
- one application-wide scheduled LLM client around the PromptKit adapter;
|
||||
- PromptKit profile-source fingerprints for checkpoint safety; and
|
||||
- redacted debug and manifest provenance at application-owned boundaries.
|
||||
|
||||
The upgrade must preserve those established responsibilities while revising
|
||||
the pinned integration contract and any behavior affected by the three
|
||||
intervening releases.
|
||||
|
||||
This roadmap is based on PromptKit's pinned release guides for
|
||||
[v0.6.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.6.0.md),
|
||||
[v0.7.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.7.0.md),
|
||||
and
|
||||
[v0.8.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/releases/v0.8.0.md),
|
||||
plus the public API and format documentation at the v0.8.0 tag.
|
||||
|
||||
## Target End State
|
||||
|
||||
- `go.mod` and `go.sum` pin PromptKit v0.8.0 without a local replacement or
|
||||
vendored copy.
|
||||
- Every maintained PromptKit prompt and profile prepares successfully under
|
||||
v0.8.0's stricter validation and source-loading rules.
|
||||
- Eligible Notarius structured completions use one PromptKit corrective call by
|
||||
default after a structurally invalid response. Operators can explicitly set
|
||||
a value from zero through three for a configured pipeline, with a more local
|
||||
LLM-backed binding override where needed.
|
||||
- PromptKit repair remains an inner operation within one Notarius stage
|
||||
attempt. It never consumes or replenishes the binding's `retries` budget.
|
||||
- A successful repaired result exposes cumulative usage and the actual repair
|
||||
count to Notarius's application-owned response and debug models. A repaired
|
||||
success is not itself a warning.
|
||||
- Exhausted PromptKit validation remains an invalid structured-output result,
|
||||
preserving the final candidate and diagnostics for debug and for any
|
||||
applicable outer Notarius stage policy. Invalid structured output is never
|
||||
accepted merely because the repair budget was exhausted.
|
||||
- Profile inheritance, the built-in Rakestrawhome backend/profile, optional
|
||||
credential behavior, and structured generation errors work through the
|
||||
existing Notarius PromptKit boundary and are accurately documented.
|
||||
- Provider-specific PromptKit types do not escape `internal/framework/llm`.
|
||||
- Checkpoint identity, effective configuration, redacted summaries, and debug
|
||||
provenance reflect every execution-affecting repair or profile change.
|
||||
- Current documentation pins and describes v0.8.0; future Notarius semantic
|
||||
validation retries remain roadmap behavior rather than being conflated with
|
||||
this dependency upgrade.
|
||||
|
||||
## Release-by-Release Adoption
|
||||
|
||||
### PromptKit v0.6.0: Correctness, Safety, And Efficiency
|
||||
|
||||
PromptKit v0.6.0 adds no public declarations, but intentionally rejects several
|
||||
formerly permissive or ambiguous inputs. The upgrade must audit Notarius's
|
||||
embedded and operator-facing integration against these rules:
|
||||
|
||||
- YAML `id` and `version` metadata, rather than filenames, define prompt and
|
||||
profile identity.
|
||||
- Prompt `content_file` paths are exact, relative, contained paths; built-in
|
||||
file artifacts must resolve to regular files.
|
||||
- execution controls, output contracts, and repair budgets must be finite and
|
||||
within their documented ranges;
|
||||
- provider endpoints must be absolute HTTP or HTTPS URLs with a host and no
|
||||
user information, query, or fragment;
|
||||
- JSON documents and successful provider responses contain exactly one value;
|
||||
- successful provider responses are bounded to 16 MiB; and
|
||||
- JSON-compatible values are bounded for depth and expansion.
|
||||
|
||||
Notarius should rely on PromptKit for these rules rather than duplicate its
|
||||
parsers or internal limits. Existing Notarius validation may retain a narrower
|
||||
application rule where it has independent value, but overlapping validation
|
||||
must agree with PromptKit and must not accept a value PromptKit will reject
|
||||
later.
|
||||
|
||||
The upgrade automatically receives operation-local schema-plan reuse,
|
||||
artifact-text memoization, improved cancellation checks, and transport error
|
||||
identity preservation. Notarius should verify these changes through its real
|
||||
adapter boundary and avoid adding a second cache or response-body layer that
|
||||
would duplicate PromptKit's ownership.
|
||||
|
||||
### PromptKit v0.7.0: Profiles, Backend Access, And Generation Errors
|
||||
|
||||
#### Profile Inheritance
|
||||
|
||||
Operator profiles may use `base_profile` to alias or selectively refine a
|
||||
built-in, fallback, or higher-precedence operator profile. Notarius must pass
|
||||
profile sources through unchanged and let PromptKit own parent lookup, merge
|
||||
rules, source precedence, cycle detection, and fully resolved prepared targets.
|
||||
|
||||
Preflight inspection must resolve inherited profiles through the same source
|
||||
and backend composition used at execution. The selected leaf profile ID remains
|
||||
the public profile identity, while effective backend, endpoint, model, and
|
||||
reasoning provenance reflect the resolved chain. Notarius must not implement a
|
||||
second inheritance parser.
|
||||
|
||||
The existing complete `dnd-extraction` fallback remains a standalone profile:
|
||||
PromptKit v0.8.0 does not provide a built-in `openai/gpt-5.6-luna` profile that
|
||||
would be an appropriate parent. Documentation should nevertheless explain how
|
||||
operators can use inheritance for environment-specific workload profiles and
|
||||
should link to PromptKit's pinned format contract rather than duplicate its
|
||||
field-by-field merge algorithm.
|
||||
|
||||
Checkpoint safety must cover inherited behavior. Operator file/directory
|
||||
digests already cover changes to definitions in those sources, fallback asset
|
||||
digests cover application parents, and the PromptKit built-in catalog marker
|
||||
must change from its v0.5.0 identity to v0.8.0 so a changed built-in parent
|
||||
cannot reuse an incompatible checkpoint.
|
||||
|
||||
#### Rakestrawhome Backend And Profile
|
||||
|
||||
PromptKit's reserved `rakestrawhome` backend and
|
||||
`rakestrawhome-gemma-4-31b` profile become available without Notarius-specific
|
||||
registration. Notarius must not register or shadow the reserved backend ID.
|
||||
Profile preflight, backend-capacity reporting, scheduling, generation, and
|
||||
provenance should work for it through the same generic paths used by OpenRouter
|
||||
and `local`.
|
||||
|
||||
The D&D default remains `dnd-extraction`; this upgrade does not silently move a
|
||||
production workload to Rakestrawhome. Operator documentation should identify
|
||||
the built-in profile as an available selection and link to PromptKit for its
|
||||
endpoint, credential environment, model, and capacity defaults.
|
||||
|
||||
#### Optional Credentials
|
||||
|
||||
An absent or blank optional `APIKeyEnv` now causes PromptKit to omit the
|
||||
`Authorization` header and send the request. Notarius must not restore the old
|
||||
failure behavior by pre-reading provider credential environment variables or
|
||||
by adding provider-specific authentication logic.
|
||||
|
||||
Profile inspection may report an explicit `APIKeyRequired` policy without
|
||||
reading the credential, and execution remains the boundary at which that
|
||||
requirement is enforced. For optional profiles, an authentication-requiring
|
||||
provider may instead return a structured 401 or 403 generation failure. The
|
||||
configuration and operations documentation must explain this distinction.
|
||||
Notarius does not currently expose PromptKit's in-memory profile-registration
|
||||
API to operators, and PromptKit's filesystem profile format does not expose
|
||||
`APIKeyRequired`; therefore Notarius must not promise that an operator profile
|
||||
can force local credential preflight. Operators should provision the named
|
||||
environment variable, while Notarius should preserve the provider's structured
|
||||
authentication failure when it is absent.
|
||||
|
||||
Notarius must continue to document mechanisms and environment-variable names,
|
||||
never secret values.
|
||||
|
||||
#### Structured Generation Errors
|
||||
|
||||
The adapter should recognize `*promptkit.GenerationError` with `errors.As` and
|
||||
translate useful information into an immutable, provider-neutral Notarius
|
||||
error classification. At minimum, retain the HTTP status code so callers and
|
||||
future retry policy can distinguish transport success with provider rejection
|
||||
from other generation failures.
|
||||
|
||||
PromptKit's provider code, type, and message accessors are bounded but remain
|
||||
untrusted and potentially sensitive. They must never appear automatically in
|
||||
ordinary CLI output, warnings, manifests, checkpoint identity, or cache data.
|
||||
If retained for an explicitly requested debug trace, they must pass through
|
||||
Notarius's known-secret and bearer redaction and remain clearly identified as
|
||||
untrusted provider diagnostics. Default error formatting should continue to
|
||||
use a bounded, redacted application-owned message.
|
||||
|
||||
Capacity and cancellation retain their current more specific classifications
|
||||
and precedence. This upgrade does not add automatic provider-error retry
|
||||
classification; it only preserves safe structured data needed for diagnosis
|
||||
and later policy.
|
||||
|
||||
### PromptKit v0.8.0: Bounded Structured-Output Repair
|
||||
|
||||
#### Default Policy
|
||||
|
||||
Every maintained production prompt whose output is consumed as structured data
|
||||
should declare one repair attempt. All current production prompts use eligible
|
||||
JSON Schema validation, so no current prompt needs a zero default merely
|
||||
because of its output mode.
|
||||
|
||||
One repair means at most one corrective generation after the initial
|
||||
candidate. PromptKit reconstructs the immutable original conversation and
|
||||
appends only the latest invalid assistant candidate and latest deterministic
|
||||
validation diagnostics. It preserves the selected target, direct session ID,
|
||||
provider-native structured-output contract, and backend capacity policy. This
|
||||
shape preserves the original cacheable prompt prefix and avoids accumulating
|
||||
unbounded failed history.
|
||||
|
||||
The default is deliberately small. A single repair captures the common case in
|
||||
which a capable model can correct malformed JSON or a schema violation after
|
||||
receiving an exact diagnostic, while bounding the extra latency and cost of a
|
||||
single structured completion.
|
||||
|
||||
#### Configuration Contract
|
||||
|
||||
The public configuration is an optional, presence-aware
|
||||
`structured_output_repair_attempts` integer at pipeline scope and at each
|
||||
LLM-backed module or validator binding. Its effective precedence is:
|
||||
|
||||
1. the binding value, when present;
|
||||
2. the pipeline value, when present; and
|
||||
3. the selected prompt's declared `repair_attempts` value.
|
||||
|
||||
The value must be from zero through three. Explicit zero disables PromptKit
|
||||
repair at that scope. A deterministic binding must reject the field because it
|
||||
cannot perform structured LLM repair. Validator bindings may use it only when
|
||||
the selected validator is LLM-backed. Shorthand module bindings continue to
|
||||
inherit the pipeline or prompt default.
|
||||
|
||||
The long, provider-neutral name is intentional: it distinguishes PromptKit's
|
||||
inner structural repair from the existing binding `retries` field, which owns
|
||||
complete stage attempts, without exposing a dependency name in generic
|
||||
pipeline contracts.
|
||||
|
||||
The effective value must survive file parsing, cloning, redacted summaries,
|
||||
pipeline resolution, and pipeline digest construction without pointer aliasing
|
||||
or loss of presence. It must affect checkpoint identity because it can change
|
||||
the selected result, latency, token usage, and provider cost.
|
||||
|
||||
#### Adapter Contract
|
||||
|
||||
The transport-neutral structured-completion request should carry an optional
|
||||
application-owned structural-repair budget. No `promptkit.OutputContract` or
|
||||
other PromptKit type may cross the adapter boundary.
|
||||
|
||||
PromptKit v0.8.0 request validation replaces the complete prompt output
|
||||
contract rather than merging one field. When Notarius has a configured
|
||||
override, the adapter must therefore inspect the selected prompt, copy its
|
||||
normalized declared format, validation mode, and schema path, change only the
|
||||
repair count, and supply that complete contract on the prepared request. A nil
|
||||
override continues to use the prompt declaration directly. Inspection and
|
||||
preparation must use the same immutable engine sources; a small adapter-local
|
||||
cache keyed by normalized prompt ID and version is acceptable but not required
|
||||
without measured need.
|
||||
|
||||
This approach prevents configuration from accidentally dropping JSON Schema
|
||||
validation, avoids duplicating schema paths in pipeline YAML, and keeps prompt
|
||||
assets authoritative for every output-contract field other than the explicit
|
||||
operator override.
|
||||
|
||||
The transport-neutral structured-completion response should report the actual
|
||||
number of PromptKit repair calls. PromptKit's returned token usage is already
|
||||
cumulative and must be passed through without re-summing it. Debug records
|
||||
should distinguish the configured budget from the actual count. Ordinary run
|
||||
manifests need not gain raw prompt or response data merely to report repairs;
|
||||
any durable aggregate should be added only if it has a clear consumer contract.
|
||||
|
||||
#### Result And Failure Semantics
|
||||
|
||||
- A valid initial candidate returns normally with zero actual repairs.
|
||||
- A valid corrected candidate returns normally with cumulative usage and its
|
||||
positive actual repair count. It does not emit a warning solely because a
|
||||
repair occurred.
|
||||
- Exhausting the repair budget returns PromptKit's final candidate and failed
|
||||
validation result. The adapter maps this to
|
||||
`ErrInvalidStructuredOutput`, preserves the response and debug material, and
|
||||
does not decode or accept the candidate.
|
||||
- An explicitly empty or whitespace-only candidate participates in the
|
||||
declared structural validation and repair flow. Missing, `null`, or
|
||||
non-string provider content remains a malformed provider response.
|
||||
- A generation failure during a corrective call is an operational generation
|
||||
failure and uses the same safe structured-error adaptation as an initial
|
||||
generation failure.
|
||||
- Context cancellation remains authoritative throughout the initial and
|
||||
corrective calls.
|
||||
|
||||
PromptKit repair happens inside one scheduled `CompleteStructured` operation.
|
||||
The Notarius scheduler holds one permit for that logical operation while
|
||||
PromptKit performs its initial and serial corrective calls; PromptKit
|
||||
reacquires its own selected-backend capacity for each corrective generation.
|
||||
Because corrective calls are serial, this cannot expand actual concurrent
|
||||
provider work beyond the number of admitted Notarius operations, but
|
||||
documentation must stop describing the Notarius permit as a separate admission
|
||||
event for every internal repair call.
|
||||
|
||||
One `CompleteStructured` invocation with effective PromptKit repair budget `R`
|
||||
may make at most `R + 1` provider calls. If one stage attempt makes `C`
|
||||
structured-completion invocations, a binding with `retries: N` has an upper
|
||||
bound of `(N + 1) * C * (R + 1)` provider calls; `C` may itself be a bounded,
|
||||
data-dependent module property, as it is for batched semantic reconciliation.
|
||||
LLM-backed validators have their own corresponding invocation counts, budgets,
|
||||
and costs. These formulas are upper bounds, not promises that every failure is
|
||||
retryable or that every attempt reaches the provider.
|
||||
|
||||
## Profile And Prompt Source Compatibility
|
||||
|
||||
The upgrade must preserve Notarius's source precedence: an operator source,
|
||||
then registered application fallback profiles, then PromptKit built-ins. A
|
||||
selected malformed definition remains authoritative and fails rather than
|
||||
falling through. Parent resolution introduced by profile inheritance observes
|
||||
that same precedence.
|
||||
|
||||
All embedded prompt manifests, shared content fragments, response schemas, and
|
||||
fallback profiles must be prepared or inspected offline under v0.8.0. The
|
||||
review should specifically catch:
|
||||
|
||||
- IDs inferred accidentally from filenames;
|
||||
- stale or escaping `content_file` paths;
|
||||
- missing or non-regular embedded artifacts;
|
||||
- repair values outside zero through three or paired with ineligible
|
||||
validation;
|
||||
- schemas or examples that are not exact single JSON documents;
|
||||
- unsupported endpoint forms; and
|
||||
- JSON-compatible variables or profile extras that exceed upstream bounds.
|
||||
|
||||
No prompt prose, schema shape, durable D&D artifact contract, or default D&D
|
||||
model should change merely to exercise the dependency. Prompt manifests should
|
||||
change only as needed to enable the adopted repair default and satisfy v0.8.0
|
||||
contracts.
|
||||
|
||||
## Provenance, Debugging, And Security
|
||||
|
||||
- Update the opaque PromptKit built-in profile-catalog identity from v0.5.0 to
|
||||
v0.8.0. Do not hash or publish PromptKit's internal catalog bytes.
|
||||
- Ensure a prompt's repair default remains covered by its existing prompt asset
|
||||
fingerprint and a configured effective override remains covered by the
|
||||
resolved pipeline digest.
|
||||
- Preserve selected leaf profile identity while recording the inherited
|
||||
effective target already exposed by PromptKit inspection and prepared
|
||||
details.
|
||||
- Add actual structural-repair count and, when useful, the configured budget to
|
||||
application-owned debug material. Token totals remain PromptKit's cumulative
|
||||
values.
|
||||
- Do not generate a warning for a successful repair. Repair exhaustion is an
|
||||
invalid-output failure, while provider rejection is a generation failure.
|
||||
- Never expose raw provider diagnostic fields without explicit debug capture
|
||||
and application redaction. Do not place them in normal errors or durable
|
||||
summaries.
|
||||
- Preserve context and transport error identity sufficiently for
|
||||
`errors.Is`-based cancellation and deadline handling after adapting the
|
||||
external error.
|
||||
|
||||
## Documentation And Examples
|
||||
|
||||
Implementation should update current-state documentation only when the new
|
||||
behavior lands:
|
||||
|
||||
- `docs/integrations/pkg-promptkit.md` must pin v0.8.0 and define the revised
|
||||
prepared-execution, repair, profile-inheritance, backend, credential, and
|
||||
error-adaptation boundary.
|
||||
- `docs/config.md` must own the repair configuration fields, precedence,
|
||||
allowed range, explicit-zero behavior, profile inheritance availability, and
|
||||
optional credential semantics.
|
||||
- `docs/operations.md` must explain structural repair cost, timeout and
|
||||
concurrency effects, credential failures, and its distinction from stage
|
||||
retries.
|
||||
- `docs/internal/llm.md` must describe adapter contract replacement, actual
|
||||
repair metadata, error adaptation, source compatibility, and scheduling.
|
||||
- `docs/internal/pipeline.md` must describe how effective repair configuration
|
||||
is resolved and how inner repair differs from outer stage attempts.
|
||||
- `docs/policy/architecture.md` should receive only the durable ownership rule:
|
||||
PromptKit owns bounded deterministic structural repair within one completion,
|
||||
while Notarius owns stage attempts and semantic validation policy. Detailed
|
||||
fields and retry formulas belong in their canonical configuration and
|
||||
operations documents.
|
||||
|
||||
Update maintained configuration examples only if the public Notarius
|
||||
configuration contract changes. A short inheritance illustration may remain in
|
||||
the configuration reference; do not create a complete example solely to copy
|
||||
PromptKit's upstream profile catalog. All upstream links must point to the
|
||||
v0.8.0 tag. Historical release or archived roadmap references should remain
|
||||
historical.
|
||||
|
||||
No ADR is required solely to pin a newer dependency. The durable separation
|
||||
between PromptKit structural repair and Notarius semantic stage retries should
|
||||
be stated in architecture documentation now; the more extensive future
|
||||
validation state machine still warrants the separate ADR already identified in
|
||||
`future.md` when that work is promoted.
|
||||
|
||||
## Validation And Acceptance Criteria
|
||||
|
||||
The implementation is complete when:
|
||||
|
||||
- the repository builds and tests against PromptKit v0.8.0 with no replacement
|
||||
directive, workspace dependency, or vendored source;
|
||||
- every maintained prompt and profile prepares or inspects successfully under
|
||||
the v0.8.0 source, path, endpoint, output-contract, and JSON-value rules;
|
||||
- an invalid first JSON Schema candidate followed by a valid correction returns
|
||||
the valid raw output, cumulative usage, and actual repair count through the
|
||||
Notarius adapter;
|
||||
- repair exhaustion returns the final raw candidate and debug material with an
|
||||
error matching `ErrInvalidStructuredOutput`;
|
||||
- a corrective generation failure retains safe generation classification and
|
||||
provider status without leaking untrusted provider detail;
|
||||
- explicit empty content follows structural validation rather than being
|
||||
misclassified by Notarius;
|
||||
- repair configuration is presence-aware, range checked, rejected on
|
||||
deterministic bindings, resolved with documented precedence, and included in
|
||||
effective pipeline identity;
|
||||
- inherited profiles resolve consistently during preflight and execution, and
|
||||
changes to any relevant operator, fallback, or built-in parent invalidate
|
||||
checkpoint reuse;
|
||||
- the Rakestrawhome built-in profile reaches generic preflight, scheduling, and
|
||||
provenance paths without application-specific registration;
|
||||
- optional missing credentials and explicitly required credentials behave as
|
||||
documented without contacting real providers in tests;
|
||||
- cancellation, timeout, backend capacity, prepared-execution snapshot,
|
||||
session ID, raw-output, debug-redaction, and existing profile provenance
|
||||
behavior remain intact;
|
||||
- maintained examples validate successfully; and
|
||||
- canonical documentation contains no active v0.5.0 pin or claim that PromptKit
|
||||
is always single-pass.
|
||||
|
||||
Tests should follow `docs/policy/testing.md`: exercise observable Notarius
|
||||
contracts with offline fake clients or `httptest` boundaries, and do not copy
|
||||
PromptKit's entire internal repair test suite or assert its exact correction
|
||||
message prose. The dependency's internal wording is not a Notarius contract.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Implementing Notarius's future feedback-aware semantic stage-retry loop.
|
||||
- Adding the D&D combat-scene semantic validator.
|
||||
- Redesigning warning policy or treating successful structural repair as a
|
||||
warning.
|
||||
- Adding provider transport retries or deciding which HTTP statuses should
|
||||
consume a stage retry.
|
||||
- Exposing PromptKit request, response, profile, validation, capacity, or error
|
||||
types outside the LLM adapter.
|
||||
- Changing durable artifact schemas, D&D prompt semantics, the D&D default
|
||||
model, or the fixed pipeline shape.
|
||||
- Reimplementing PromptKit profile inheritance, schema validation, response
|
||||
bounds, repair conversations, backend admission, or provider parsing inside
|
||||
Notarius.
|
||||
|
||||
## Decisions
|
||||
|
||||
### 1. Default Structured-Output Repair Budget
|
||||
|
||||
**Decision: default to one repair attempt.** Set every maintained
|
||||
eligible production prompt to `repair_attempts: 1`. One corrective call is a
|
||||
strong fit for Notarius because every current production LLM response has a
|
||||
strict JSON Schema contract, smaller cost-effective models are a deliberate
|
||||
deployment target, and a precise structural diagnostic often makes one retry
|
||||
materially more successful. The budget is paid only after a structurally
|
||||
invalid candidate and remains tightly bounded.
|
||||
|
||||
**Alternative considered: retain zero by default.** This preserves single-pass
|
||||
cost and latency and requires operators to opt in. It is preferable for an
|
||||
environment where every additional request is expensive or where upstream
|
||||
provider-native schema enforcement already produces negligible invalid output.
|
||||
It is less suitable as the Notarius default because one malformed response can
|
||||
otherwise discard substantial completed pipeline work.
|
||||
|
||||
**Alternative considered: default to two.** This may improve recovery for
|
||||
weak models, but it doubles the worst-case corrective cost relative to the
|
||||
selected default and compounds with outer stage retries. It should be an
|
||||
operator choice supported by configuration, not the initial default, unless
|
||||
observational evidence shows that the second correction has a worthwhile
|
||||
marginal success rate.
|
||||
|
||||
### 2. Repair Override Scope
|
||||
|
||||
**Decision: support both pipeline and LLM-backed binding overrides.** Use
|
||||
the presence-aware `structured_output_repair_attempts` field and precedence
|
||||
defined above. A pipeline value provides the convenient one-line control the
|
||||
operator requested, while a binding value permits an expensive normalizer or
|
||||
future LLM-backed validator to use a deliberately different budget. This
|
||||
mirrors Notarius's established pipeline/binding profile inheritance and scales
|
||||
without editing embedded prompts.
|
||||
|
||||
**Alternative considered: support only a pipeline override.** This is smaller to
|
||||
implement and document and still permits global enablement or disablement for
|
||||
one pipeline. Its drawback is that one exceptional prompt cannot opt out or
|
||||
request a larger budget without changing an embedded asset for every pipeline.
|
||||
|
||||
**Alternative considered: expose one global value under the top-level
|
||||
`promptkit` configuration.** This makes client construction simple, but applies
|
||||
the same budget to unrelated pipelines and leaks an execution policy into the
|
||||
dependency configuration block. It is less compositional than pipeline-owned
|
||||
policy and therefore not recommended.
|
||||
|
||||
### 3. Retention Of Provider-Supplied Generation Details
|
||||
|
||||
**Decision: retain status in the application-owned error contract and
|
||||
retain redacted provider code, type, and message only in explicitly requested
|
||||
debug traces.** Status is useful for diagnosis and future retry policy without
|
||||
usually containing sensitive data. The other fields can materially explain a
|
||||
400 response but may echo request or schema content, so they belong only in the
|
||||
already-sensitive debug surface after Notarius redaction.
|
||||
|
||||
**Alternative considered: retain only HTTP status and discard all provider fields.**
|
||||
This is the safest and smallest policy and still improves typed failure
|
||||
handling. It sacrifices potentially decisive provider diagnostics, leaving an
|
||||
operator with less information when a provider returns a terse status and the
|
||||
problem cannot be reproduced easily.
|
||||
|
||||
**Alternative considered: include bounded provider code and type in normal
|
||||
errors while keeping message debug-only.** Codes and types are often stable and
|
||||
less sensitive than messages, but PromptKit explicitly classifies every
|
||||
provider field as untrusted. Promoting them to ordinary output creates a
|
||||
disclosure and compatibility burden that is not currently justified.
|
||||
@@ -1,282 +0,0 @@
|
||||
# Source-Only Releases
|
||||
|
||||
## Status
|
||||
|
||||
Implemented. Creating the first release under this procedure remains a
|
||||
separate maintainer operation.
|
||||
|
||||
## Purpose
|
||||
|
||||
Define a repeatable, guarded release process for Notarius without taking on a
|
||||
binary-distribution system that its current operator audience does not need.
|
||||
The process should make an exact source revision, its compatibility impact,
|
||||
and its validation status easy to identify while keeping installation in the
|
||||
hands of technically capable operators and deployment automation.
|
||||
|
||||
The model is adapted from Weatherreporter's release procedure, but its target
|
||||
is deliberately narrower: an immutable source tag and checked-in release note
|
||||
are the release. Notarius does not publish executable archives or support
|
||||
Windows as part of this work.
|
||||
|
||||
## Release Model
|
||||
|
||||
Notarius releases come from commits on `main` and use stable semantic-version
|
||||
tags in the form `vMAJOR.MINOR.PATCH`. Prerelease tags are not part of the
|
||||
initial process.
|
||||
|
||||
Every release has one nonempty, version-matched note at
|
||||
`docs/releases/<tag>.md`. The note and every affected current-state document
|
||||
must be present in the tagged commit. The Git tag and checked-in note together
|
||||
are the durable release record; no separately editable release page is
|
||||
required.
|
||||
|
||||
Published tags are immutable. A maintainer must never move, reuse, or delete a
|
||||
published tag. If a published candidate is defective, the correction is made
|
||||
on `main` and released under a new patch version. An unpublished local tag may
|
||||
be deleted when candidate inspection finds a problem before any remote push.
|
||||
|
||||
Before `v1.0.0`, a minor release may intentionally change a documented CLI,
|
||||
configuration, durable artifact, integration, or operating contract when its
|
||||
release note explains the impact and required operator action. A patch release
|
||||
must not intentionally break those documented contracts within its minor
|
||||
line.
|
||||
|
||||
The existing `v0.1.0`, `v0.2.0`, and `v0.3.0` tags remain unchanged. They
|
||||
predate this procedure and do not need retrospective release notes. The first
|
||||
release made under this process establishes the release-note series.
|
||||
|
||||
## Source-Only Distribution
|
||||
|
||||
Notarius does not publish release binaries, archives, installers, container
|
||||
images, package-manager entries, checksum files, or signatures. A release tag
|
||||
is suitable for Go-native installation and for an operator-controlled build
|
||||
from an exact checkout.
|
||||
|
||||
The primary installation form is:
|
||||
|
||||
```sh
|
||||
GOWORK=off go install \
|
||||
gitea.maximumdirect.net/eric/notarius/cmd/notarius@vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
Operator documentation should also describe cloning the repository, checking
|
||||
out the tag in detached-head state, and building `./cmd/notarius` with the Go
|
||||
version declared by `go.mod`. Private-module authentication and `GOPRIVATE`
|
||||
configuration belong to the operator environment and must be documented by
|
||||
mechanism rather than with real credentials.
|
||||
|
||||
Consumers such as Narratio should pin the desired Notarius tag in provisioning
|
||||
or deployment configuration. They must continue to decide runtime
|
||||
compatibility from Notarius's published receipt and artifact schema contracts,
|
||||
not merely from the executable's product version.
|
||||
|
||||
Packaged binaries may be reconsidered if distribution demand, installation
|
||||
friction, or a broader user audience justifies their build, signing, retention,
|
||||
and platform-support costs. They are not a prerequisite for a disciplined
|
||||
release process.
|
||||
|
||||
## Platform Policy
|
||||
|
||||
Linux is the supported deployment platform. Release validation must run the
|
||||
test suite and the release build on Linux and must confirm that the command
|
||||
builds with `CGO_ENABLED=0` for Linux `amd64` and `arm64`.
|
||||
|
||||
macOS is a best-effort development and testing platform. Release validation
|
||||
should confirm that the command cross-compiles with `CGO_ENABLED=0` for Darwin
|
||||
`amd64` and `arm64`, but the project does not promise packaged artifacts or a
|
||||
separate runtime test environment for those targets.
|
||||
|
||||
Windows is unsupported. The release process must not require Windows builds,
|
||||
Windows-specific compatibility work, or Windows documentation. Platform-
|
||||
specific implementation may intentionally use Unix facilities when they are
|
||||
important to Notarius's filesystem safety and operational model. Any later
|
||||
decision to support Windows requires its own feature scope and validation
|
||||
policy.
|
||||
|
||||
## Version Reporting
|
||||
|
||||
Add a root `notarius --version` interface for deployment diagnostics. It
|
||||
prints exactly one line:
|
||||
|
||||
```text
|
||||
notarius vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
when the build has a valid release version, and:
|
||||
|
||||
```text
|
||||
notarius development
|
||||
```
|
||||
|
||||
when no release version is available.
|
||||
|
||||
The implementation must obtain the main-module version from Go build
|
||||
information so `go install ...@vMAJOR.MINOR.PATCH` reports the selected tag. It
|
||||
must also accept an optional link-time version override so controlled builds
|
||||
and release CI can identify an exact tag from a checkout. The override must be
|
||||
validated and must not silently turn arbitrary text into a release version.
|
||||
Ordinary unversioned checkout builds remain `development`; the release process
|
||||
must not modify a tracked source constant for each release.
|
||||
|
||||
Version reporting is an informational product interface. It does not replace
|
||||
receipt, configuration, prompt, or artifact schema versioning, and it must not
|
||||
be used as the sole downstream compatibility check.
|
||||
|
||||
## Release Notes
|
||||
|
||||
Each new `docs/releases/<tag>.md` document has this minimum structure:
|
||||
|
||||
```markdown
|
||||
# Notarius vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
The note should concisely explain the release's purpose, compatibility with the
|
||||
preceding release, operator actions, and material user-visible, operational,
|
||||
integration, and maintainer-visible changes. It should link to canonical
|
||||
current-state documentation for exact contracts rather than duplicating those
|
||||
contracts.
|
||||
|
||||
Release notes are durable historical summaries. They must not contain
|
||||
credentials, private infrastructure detail, sensitive campaign material, or
|
||||
claims that are not true of the tagged candidate. A release note does not
|
||||
excuse stale current-state documentation; affected canonical documents are
|
||||
updated in the same candidate.
|
||||
|
||||
## Candidate Validation
|
||||
|
||||
The release procedure must provide copyable POSIX-shell guards that validate
|
||||
the release version, release-note filename and heading, required note sections,
|
||||
repository state, and module hygiene. Validation must be run from the Notarius
|
||||
repository root with Go workspace behavior disabled.
|
||||
|
||||
At minimum, a candidate must pass:
|
||||
|
||||
- no tracked `go.work` or `go.work.sum`, no vendored tree, and no `replace`
|
||||
directive in `go.mod`;
|
||||
- `GOWORK=off go test -count=1 ./...`;
|
||||
- `GOWORK=off go test -race -count=1 ./...`;
|
||||
- `GOWORK=off go vet ./...`;
|
||||
- `GOWORK=off go build ./...`;
|
||||
- `GOWORK=off go mod tidy -diff`;
|
||||
- `gofmt` verification for every tracked Go file;
|
||||
- `git diff --check` and `git diff --cached --check`;
|
||||
- validation of both maintained D&D configuration examples with their selected
|
||||
pipeline;
|
||||
- Linux `amd64` and `arm64` static command builds;
|
||||
- best-effort Darwin `amd64` and `arm64` static command builds; and
|
||||
- a focused manual or automated check that every added or changed local
|
||||
Markdown link resolves.
|
||||
|
||||
The candidate review also checks for generated binaries, test output,
|
||||
credentials, temporary files, module replacements, vendored dependencies, and
|
||||
other unintended source-control content. Tests remain offline and do not call
|
||||
an LLM provider or require live credentials.
|
||||
|
||||
## Candidate Publication
|
||||
|
||||
The release procedure must guard the exact commit immediately before tagging.
|
||||
It requires:
|
||||
|
||||
- the current branch is `main`;
|
||||
- the worktree and index are clean;
|
||||
- the candidate commit has been pushed and exactly matches `origin/main`;
|
||||
- the matching release note exists in that commit;
|
||||
- no local or remote tag already uses the selected version; and
|
||||
- the substantive release checks have passed for that exact candidate.
|
||||
|
||||
The maintainer records the exact candidate commit, creates a lightweight tag
|
||||
bound explicitly to that commit, verifies the local tag target, and pushes only
|
||||
that tag ref. The procedure must not recommend `git push --tags`.
|
||||
|
||||
After publication, the maintainer verifies that the remote tag resolves to the
|
||||
guarded commit and that the release note can be read from the tagged tree. A
|
||||
fresh temporary checkout or `go install ...@<tag>` must build successfully, and
|
||||
the resulting command must report the expected version through `--version`.
|
||||
|
||||
## Validation-Only Release Automation
|
||||
|
||||
Add a tag-triggered Woodpecker pipeline that validates source releases without
|
||||
publishing artifacts. It should:
|
||||
|
||||
- accept only stable semantic-version tags;
|
||||
- require the version-matched release note;
|
||||
- run the same substantive module, test, race, vet, build, formatting, and
|
||||
whitespace checks as the documented local procedure;
|
||||
- validate the maintained configuration examples;
|
||||
- perform the supported and best-effort cross-build checks; and
|
||||
- verify a release-version build's `notarius --version` output on the CI host.
|
||||
|
||||
The pipeline must not upload binaries, create archives or checksums, create or
|
||||
edit a Gitea release object, or require a release API token. Local guards remain
|
||||
authoritative before tag publication because CI begins only after the tag is
|
||||
already remote.
|
||||
|
||||
If tag validation fails, preserve the published tag, fix the cause on `main`,
|
||||
select a new patch version, and repeat the full process. Do not weaken tag
|
||||
immutability merely because the release contains source rather than binaries.
|
||||
|
||||
## Documentation Ownership
|
||||
|
||||
In the target state:
|
||||
|
||||
- `docs/release.md` owns the maintainer release procedure, commands, ordering,
|
||||
publication checks, and failure recovery;
|
||||
- `docs/releases/` owns one historical summary per release made under the new
|
||||
process;
|
||||
- `docs/cli.md` owns the `--version` contract;
|
||||
- `README.md` owns the shortest source-installation example and links to the
|
||||
release procedure where useful;
|
||||
- `docs/development.md` routes release preparation, tagging, and verification
|
||||
work to `docs/release.md`;
|
||||
- `docs/policy/documentation.md` assigns canonical ownership to the release
|
||||
procedure and release notes;
|
||||
- `docs/policy/architecture.md` records Linux support, best-effort macOS
|
||||
development, unsupported Windows, and source-only distribution only if those
|
||||
are judged durable development invariants rather than release mechanics; and
|
||||
- `docs/operations.md` describes only installation or deployment consequences
|
||||
relevant to operators and links to canonical CLI and release contracts.
|
||||
|
||||
Current-state documentation must not describe the new release process,
|
||||
`--version`, or automated validation until the corresponding behavior exists.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A maintainer can prepare, validate, tag, publish, and verify a source release
|
||||
by following `docs/release.md` without relying on undocumented knowledge.
|
||||
- Every new release has an immutable semantic-version tag and matching
|
||||
checked-in release note in the tagged commit.
|
||||
- The guarded candidate is clean, synchronized with `origin/main`, and passes
|
||||
the documented substantive checks before tagging.
|
||||
- Tag-triggered CI independently validates the published source and never
|
||||
publishes binary artifacts.
|
||||
- `go install` of a tagged version succeeds and `notarius --version` reports
|
||||
that version; ordinary unversioned builds report `development`.
|
||||
- Linux is the documented supported deployment platform, macOS has a
|
||||
best-effort development build check, and Windows is explicitly unsupported.
|
||||
- Downstream compatibility remains based on durable Notarius contracts rather
|
||||
than the product version alone.
|
||||
- Existing pre-procedure tags remain untouched and require no invented release
|
||||
history.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Publishing executable archives, installers, container images, checksums,
|
||||
signatures, or package-manager entries.
|
||||
- Supporting or cross-compiling for Windows.
|
||||
- Creating or maintaining a mutable Gitea release page.
|
||||
- Supporting prerelease tag syntax in the initial procedure.
|
||||
- Automating version selection, release-note authorship, commits, or tag
|
||||
creation.
|
||||
- Retrospectively creating release notes for `v0.1.0` through `v0.3.0`.
|
||||
- Treating a product version as a substitute for receipt, configuration,
|
||||
prompt, or artifact schema compatibility.
|
||||
7
go.mod
7
go.mod
@@ -3,9 +3,14 @@ module gitea.maximumdirect.net/eric/notarius
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.40.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@@ -1,5 +1,9 @@
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0 h1:NGd9hDLu0UMxKbvittMrqM5Ua94eFb+kOE7UIir8l08=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.8.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
|
||||
@@ -21,6 +21,10 @@ import (
|
||||
|
||||
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||
|
||||
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
|
||||
|
||||
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
|
||||
|
||||
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
@@ -67,28 +71,39 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
|
||||
}
|
||||
|
||||
wantWarningReasons := []string{
|
||||
wantDiagnosticReasons := []string{
|
||||
spellnormalize.ReasonCodeSpellNameCanonicalized,
|
||||
spellnormalize.ReasonCodeSourceReferencesNormalized,
|
||||
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
|
||||
"spell_not_near_source",
|
||||
}
|
||||
gotWarningReasons := make([]string, len(output.Warnings))
|
||||
for index, warning := range output.Warnings {
|
||||
gotWarningReasons[index] = warning.ReasonCode
|
||||
gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
|
||||
for index, group := range output.Diagnostics.Groups {
|
||||
gotDiagnosticReasons[index] = group.ReasonCode
|
||||
}
|
||||
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
|
||||
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
|
||||
if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
|
||||
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
|
||||
}
|
||||
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
|
||||
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
|
||||
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
|
||||
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
|
||||
}
|
||||
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
|
||||
if len(warningsFile.Groups) != 0 {
|
||||
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
|
||||
}
|
||||
diagnosticsFile := decodeAssembledOutput[struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
GroupCount int `json:"group_count"`
|
||||
OccurrenceCount int `json:"occurrence_count"`
|
||||
Truncated bool `json:"truncated"`
|
||||
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "diagnostics.json")
|
||||
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
|
||||
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
|
||||
}
|
||||
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
||||
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
||||
@@ -101,6 +116,30 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
|
||||
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
|
||||
}
|
||||
correction := extractor.correctionSnapshot()
|
||||
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
|
||||
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
||||
var normalizeChain *pipeline.ResolvedValidatorChain
|
||||
@@ -127,15 +166,16 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
||||
}
|
||||
for _, warning := range output.Warnings {
|
||||
if warning.ReasonCode == "spell_not_near_source" {
|
||||
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
if group.ReasonCode == "spell_not_near_source" {
|
||||
t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
|
||||
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
||||
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
@@ -161,12 +201,12 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
|
||||
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
||||
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
|
||||
t.Fatalf("warnings = %#v, want terminal normalize catalog warning", output.Warnings)
|
||||
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
|
||||
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) {
|
||||
func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
@@ -190,14 +230,20 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
|
||||
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
|
||||
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
|
||||
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings)
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
|
||||
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
|
||||
}
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings)
|
||||
if len(warningsFile.Groups) != 0 {
|
||||
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
|
||||
}
|
||||
diagnosticsFile := decodeAssembledOutput[struct {
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "diagnostics.json")
|
||||
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
|
||||
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,6 +252,47 @@ type assembledSpellPipelineOptions struct {
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
|
||||
t.Helper()
|
||||
components := productionTestComponents(t)
|
||||
extractor := &assembledCorrectingSpellExtractor{}
|
||||
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||
Key: assembledCorrectingSpellExtractorKey,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.spell_casts"},
|
||||
ArtifactKind: dnd.SpellListKind,
|
||||
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register correcting extractor: %v", err)
|
||||
}
|
||||
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
|
||||
return assembledDirectSpellValidator{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register direct spell validator: %v", err)
|
||||
}
|
||||
|
||||
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
|
||||
extract.Retries = 1
|
||||
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
|
||||
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||
ID: "assembled-dnd-correcting-spells",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
|
||||
},
|
||||
Output: pipeline.Binding("json"),
|
||||
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
return components.registries, resolved, extractor
|
||||
}
|
||||
|
||||
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||
t.Helper()
|
||||
components := productionTestComponents(t)
|
||||
@@ -251,6 +338,69 @@ type assembledSpellExtractor struct {
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
type assembledCorrectingSpellExtractor struct {
|
||||
mu sync.Mutex
|
||||
correction *contracts.SemanticCorrection
|
||||
}
|
||||
|
||||
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
|
||||
|
||||
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
if req.Source == nil || req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
|
||||
}
|
||||
response := `{"spell":"accepted"}`
|
||||
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||
if req.Chunk.Index == 0 && req.Correction == nil {
|
||||
response = `{"spell":"Mysterious Burst"}`
|
||||
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||
}
|
||||
if req.Chunk.Index == 0 && req.Correction != nil {
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.correction = correction
|
||||
e.mu.Unlock()
|
||||
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||
}
|
||||
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
correction, err := contracts.CloneSemanticCorrection(e.correction)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return correction
|
||||
}
|
||||
|
||||
type assembledDirectSpellValidator struct{}
|
||||
|
||||
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
|
||||
|
||||
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
||||
for _, cast := range req.Value.SpellCasts {
|
||||
if cast.Spell == "Mysterious Burst" {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
|
||||
}
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
||||
|
||||
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
@@ -296,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
|
||||
return append([]int(nil), e.chunkIndexes...)
|
||||
}
|
||||
|
||||
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
|
||||
count := 0
|
||||
for _, group := range groups {
|
||||
count += group.OccurrenceCount
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
|
||||
@@ -396,8 +396,12 @@ func (client *enemyEventLLMClient) CompleteStructured(ctx context.Context, reque
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||
}
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, request)
|
||||
client.requests = append(client.requests, snapshot)
|
||||
client.mu.Unlock()
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: request.ProfileID}, nil
|
||||
}
|
||||
@@ -408,7 +412,11 @@ func (client *enemyEventLLMClient) requestsFor(promptID string) []contracts.Stru
|
||||
var requests []contracts.StructuredCompletionRequest
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
requests = append(requests, request)
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
requests = append(requests, snapshot)
|
||||
}
|
||||
}
|
||||
return requests
|
||||
|
||||
@@ -91,8 +91,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
||||
if !reflect.DeepEqual(durable, want) {
|
||||
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
|
||||
if len(output.Diagnostics.Groups) != 0 {
|
||||
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -239,10 +239,10 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
||||
}
|
||||
warnings := readProductionJSON[struct {
|
||||
Warnings []json.RawMessage `json:"warnings"`
|
||||
Groups []json.RawMessage `json:"groups"`
|
||||
}](t, filepath.Join(runRoot, "warnings.json"))
|
||||
if len(warnings.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
|
||||
if len(warnings.Groups) != 0 {
|
||||
t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -343,6 +343,9 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
Inputs []struct {
|
||||
Name string `yaml:"name"`
|
||||
} `yaml:"inputs"`
|
||||
Messages []struct {
|
||||
Role string `yaml:"role"`
|
||||
} `yaml:"messages"`
|
||||
}
|
||||
preparedPrompts := 0
|
||||
if err := fs.WalkDir(promptFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
@@ -360,6 +363,11 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
if err := yaml.Unmarshal(data, &prompt); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, message := range prompt.Messages {
|
||||
if message.Role != promptkit.RoleSystem && message.Role != promptkit.RoleUser {
|
||||
return fmt.Errorf("production prompt %q uses role %q, want system or user", prompt.ID, message.Role)
|
||||
}
|
||||
}
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(prompt.Inputs))
|
||||
for _, input := range prompt.Inputs {
|
||||
inputs[input.Name] = promptkit.Inline(`{}`)
|
||||
@@ -844,10 +852,10 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
|
||||
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
|
||||
}
|
||||
warnings := readProductionJSON[struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
|
||||
if len(warnings.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want none", warnings.Warnings)
|
||||
if len(warnings.Groups) != 0 {
|
||||
t.Fatalf("warnings = %#v, want none", warnings.Groups)
|
||||
}
|
||||
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
|
||||
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
|
||||
@@ -1091,8 +1099,12 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||
}
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, req)
|
||||
client.requests = append(client.requests, snapshot)
|
||||
client.mu.Unlock()
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||
}
|
||||
@@ -1103,7 +1115,11 @@ func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.
|
||||
var requests []contracts.StructuredCompletionRequest
|
||||
for _, req := range client.requests {
|
||||
if req.PromptID == promptID {
|
||||
requests = append(requests, req)
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
requests = append(requests, snapshot)
|
||||
}
|
||||
}
|
||||
return requests
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -14,20 +15,16 @@ import (
|
||||
|
||||
func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
selector string
|
||||
only []string
|
||||
wantStage pipeline.ModuleStage
|
||||
wantLane string
|
||||
wantSlot string
|
||||
name string
|
||||
selector string
|
||||
want []string
|
||||
}{
|
||||
{name: "flat", selector: "alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "chunk", selector: "chunk.chunk-slot", wantStage: pipeline.StageChunk, wantSlot: "chunk-slot"},
|
||||
{name: "merge", selector: "merge.alpha-merge", only: []string{"alpha"}, wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane", selector: "alpha.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane extract", selector: "alpha.extract.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane merge", selector: "alpha.merge.alpha-merge", wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", wantStage: pipeline.StageNormalize, wantLane: "alpha", wantSlot: "alpha-normalize"},
|
||||
{name: "pipeline", selector: "shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared", "beta.extract.shared", "beta.merge.shared", "beta.normalize.shared"}},
|
||||
{name: "chunk", selector: "chunk.chunk-slot", want: []string{"chunk.chunk-slot"}},
|
||||
{name: "lane", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
|
||||
{name: "lane extract", selector: "alpha.extract.alpha-slot", want: []string{"alpha.extract.alpha-slot"}},
|
||||
{name: "lane merge", selector: "alpha.merge.alpha-merge", want: []string{"alpha.merge.alpha-merge"}},
|
||||
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", want: []string{"alpha.normalize.alpha-normalize"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -37,70 +34,135 @@ func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve selector: %v", err)
|
||||
}
|
||||
if len(overrides) != 1 {
|
||||
t.Fatalf("overrides = %#v, want one binding", overrides)
|
||||
if got := referenceContractBindingLabels(overrides); !slices.Equal(got, tt.want) {
|
||||
t.Fatalf("binding targets = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
got := overrides[0]
|
||||
if got.Stage != tt.wantStage || got.LaneID != tt.wantLane || got.SlotName != tt.wantSlot || got.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding = %#v, want %s/%s/%s from CLI", got, tt.wantStage, tt.wantLane, tt.wantSlot)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectAmbiguityWithSpecificSuggestions(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
want []string
|
||||
}{
|
||||
{name: "flat shared slot", selector: "shared", want: []string{"alpha.extract.shared", "beta.extract.shared"}},
|
||||
{name: "lane shared slot", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
|
||||
{name: "all mergers", selector: "merge.shared", want: []string{"alpha.merge.shared", "beta.merge.shared"}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("resolve selector succeeded, want ambiguity error")
|
||||
}
|
||||
for _, fragment := range tt.want {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %q, want suggestion %q", err, fragment)
|
||||
for _, binding := range overrides {
|
||||
if binding.Source != "reference.txt" || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding = %#v, want CLI source", binding)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRespectSelectedLanesBeforeMaterialization(t *testing.T) {
|
||||
func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
requests := []cliReferenceRequest{
|
||||
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-first.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-final.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "lane.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "binding.txt"},
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
want := map[string]string{
|
||||
"alpha.extract.shared": "binding.txt",
|
||||
"alpha.merge.shared": "lane.txt",
|
||||
"alpha.normalize.shared": "lane.txt",
|
||||
"beta.extract.shared": "pipeline-final.txt",
|
||||
"beta.merge.shared": "pipeline-final.txt",
|
||||
"beta.normalize.shared": "pipeline-final.txt",
|
||||
}
|
||||
for _, binding := range overrides {
|
||||
label := referenceContractBindingLabel(binding)
|
||||
if binding.Source != want[label] {
|
||||
t.Fatalf("binding %s source = %q, want %q", label, binding.Source, want[label])
|
||||
}
|
||||
delete(want, label)
|
||||
}
|
||||
if len(want) != 0 {
|
||||
t.Fatalf("missing bindings: %#v", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
|
||||
catalog := catalogFromRegistries(productionTestComponents(t).registries)
|
||||
sources := map[string]string{
|
||||
"party": "/references/party.txt",
|
||||
"players": "/references/players.txt",
|
||||
"glossary": "/references/glossary.txt",
|
||||
"spell_catalog": "/references/spells.json",
|
||||
}
|
||||
requests := make([]cliReferenceRequest, 0, len(sources))
|
||||
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
|
||||
requests = append(requests, cliReferenceRequest{
|
||||
Selector: mustParseReferenceSelector(t, slot, "--reference"),
|
||||
Source: sources[slot],
|
||||
})
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand complete D&D references: %v", err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
|
||||
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
|
||||
for _, binding := range overrides {
|
||||
actual[referenceContractBindingLabel(binding)] = binding
|
||||
}
|
||||
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matched := make(map[string]int, len(sources))
|
||||
for _, target := range targets {
|
||||
for slot, sourcePath := range sources {
|
||||
if _, ok := target.slots[slot]; !ok {
|
||||
continue
|
||||
}
|
||||
matched[slot]++
|
||||
label := targetLabel(target) + "." + slot
|
||||
binding, ok := actual[label]
|
||||
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
for slot := range sources {
|
||||
if matched[slot] < 2 {
|
||||
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
|
||||
}
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
|
||||
t.Fatalf("resolve complete D&D CLI references: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
only []string
|
||||
want string
|
||||
}{
|
||||
{name: "unselected lane", selector: "beta.extract.beta-slot", want: `reference lane "beta" is not selected`},
|
||||
{name: "pipeline slot", selector: "missing", want: `reference slot "missing" is not declared by any selected target`},
|
||||
{name: "lane slot", selector: "alpha.missing", want: `reference slot "missing" is not declared by selected lane "alpha"`},
|
||||
{name: "binding slot", selector: "alpha.extract.missing", want: `reference slot "missing" is not declared`},
|
||||
{name: "former merge shorthand", selector: "merge.shared", want: `reference lane "merge" is not selected`},
|
||||
{name: "unselected lane", selector: "beta.extract.beta-slot", only: []string{"alpha"}, want: `reference lane "beta" is not selected`},
|
||||
{name: "unknown lane", selector: "missing.extract.beta-slot", want: `reference lane "missing" is not selected`},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", []string{"alpha"}, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
|
||||
selector := mustParseReferenceSelector(t, tt.selector, "--reference")
|
||||
_, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || strings.Contains(err.Error(), "missing.txt") {
|
||||
t.Fatalf("error = %v, want selection failure before file access", err)
|
||||
t.Fatalf("error = %v, want selection failure containing %q before file access", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -131,40 +193,50 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceOverridesUseFinalExactTargetBinding(t *testing.T) {
|
||||
func TestReferenceBindAndUnbindSpecificity(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
alphaShared, err := parseReferenceSelector("alpha.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
betaShared, err := parseReferenceSelector("beta.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{
|
||||
{Selector: alphaShared, Source: "alpha-first.txt"},
|
||||
{Selector: alphaShared, Source: "alpha-final.txt"},
|
||||
{Selector: betaShared, Source: "beta-only.txt"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve pipeline: %v", err)
|
||||
}
|
||||
alpha := referenceContractLane(t, effective.ResolvedPipeline, "alpha")
|
||||
beta := referenceContractLane(t, effective.ResolvedPipeline, "beta")
|
||||
if source := referenceContractBindingSource(alpha.ExtractReferences.Bindings, "shared"); source != "alpha-final.txt" {
|
||||
t.Fatalf("alpha shared source = %q, want final exact-target override", source)
|
||||
}
|
||||
if source := referenceContractBindingSource(beta.ExtractReferences.Bindings, "shared"); source != "beta-only.txt" {
|
||||
t.Fatalf("beta shared source = %q, want target-specific override", source)
|
||||
}
|
||||
t.Run("specific unbind carves out broad binding", func(t *testing.T) {
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "shared.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--without-reference")}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := referenceContractBindingLabels(overrides); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
|
||||
t.Fatalf("overrides = %#v, want all shared targets except alpha extract", got)
|
||||
}
|
||||
if got := referenceContractUnbindLabels(unbinds); !slices.Equal(got, []string{"alpha.extract.shared"}) {
|
||||
t.Fatalf("unbinds = %#v, want alpha extract", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("specific binding restores broad unbind", func(t *testing.T) {
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "alpha.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "shared", "--without-reference")}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := referenceContractBindingLabels(overrides); !slices.Equal(got, []string{"alpha.extract.shared"}) {
|
||||
t.Fatalf("overrides = %#v, want alpha extract", got)
|
||||
}
|
||||
if got := referenceContractUnbindLabels(unbinds); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
|
||||
t.Fatalf("unbinds = %#v, want all shared targets except alpha extract", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("same specificity conflicts", func(t *testing.T) {
|
||||
_, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "alpha.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--without-reference")}},
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "same specificity") {
|
||||
t.Fatalf("error = %v, want same-specificity conflict", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReferenceUnbindsRemoveOptionalAndProtectRequiredSlots(t *testing.T) {
|
||||
@@ -257,6 +329,52 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
profile := cfg.Pipelines["demo"]
|
||||
alpha := profile.Artifacts["alpha"]
|
||||
beta := profile.Artifacts["beta"]
|
||||
alpha.Extract.References["shared"] = pipeline.GeneratedReference("produce", "beta")
|
||||
profile.Artifacts = nil
|
||||
profile.Steps = []pipeline.PipelineStepProfile{
|
||||
{ID: "produce", Artifacts: map[string]pipeline.ArtifactLaneProfile{"beta": beta}},
|
||||
{ID: "consume", Artifacts: map[string]pipeline.ArtifactLaneProfile{"alpha": alpha}},
|
||||
}
|
||||
cfg.Pipelines["demo"] = profile
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
|
||||
t.Run("binding conflicts before file access", func(t *testing.T) {
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{
|
||||
Selector: mustParseReferenceSelector(t, "shared", "--reference"),
|
||||
Source: filepath.Join(t.TempDir(), "never-read.json"),
|
||||
}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand CLI reference: %v", err)
|
||||
}
|
||||
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") {
|
||||
t.Fatalf("resolve error = %v, want generated/external conflict before file access", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unbind leaves generated source intact", func(t *testing.T) {
|
||||
_, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, nil, []cliReferenceUnbindRequest{{
|
||||
Selector: mustParseReferenceSelector(t, "shared", "--without-reference"),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("expand CLI unbind: %v", err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceUnbinds: unbinds})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve generated reference with CLI unbind: %v", err)
|
||||
}
|
||||
binding := referenceContractFindBinding(referenceContractLane(t, effective.ResolvedPipeline, "alpha").ExtractReferences.Bindings, "shared")
|
||||
if binding == nil || binding.Artifact == nil || binding.Artifact.Step != "produce" || binding.Artifact.Lane != "beta" {
|
||||
t.Fatalf("generated binding = %#v, want preserved produce/beta handoff", binding)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReferenceTargetLookupUsesArtifactVariantsAndReportsMissingContext(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
full := referenceContractCatalog(t, true, true)
|
||||
@@ -368,7 +486,7 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
|
||||
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
if includeBetaMerger {
|
||||
@@ -444,6 +562,42 @@ func referenceContractBindingSource(bindings []pipeline.ReferenceBinding, slot s
|
||||
return ""
|
||||
}
|
||||
|
||||
func mustParseReferenceSelector(t *testing.T, value, flagName string) cliReferenceSelector {
|
||||
t.Helper()
|
||||
selector, err := parseReferenceSelector(value, flagName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return selector
|
||||
}
|
||||
|
||||
func referenceContractBindingLabels(bindings []pipeline.ReferenceBinding) []string {
|
||||
labels := make([]string, 0, len(bindings))
|
||||
for _, binding := range bindings {
|
||||
labels = append(labels, referenceContractBindingLabel(binding))
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func referenceContractBindingLabel(binding pipeline.ReferenceBinding) string {
|
||||
if binding.Stage == pipeline.StageChunk {
|
||||
return "chunk." + binding.SlotName
|
||||
}
|
||||
return binding.LaneID + "." + string(binding.Stage) + "." + binding.SlotName
|
||||
}
|
||||
|
||||
func referenceContractUnbindLabels(unbinds []pipeline.ReferenceUnbind) []string {
|
||||
labels := make([]string, 0, len(unbinds))
|
||||
for _, unbind := range unbinds {
|
||||
labels = append(labels, referenceContractBindingLabel(pipeline.ReferenceBinding{
|
||||
Stage: unbind.Stage,
|
||||
LaneID: unbind.LaneID,
|
||||
SlotName: unbind.SlotName,
|
||||
}))
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func referenceContractFindBinding(bindings []pipeline.ReferenceBinding, slot string) *pipeline.ReferenceBinding {
|
||||
for i := range bindings {
|
||||
if bindings[i].SlotName == slot {
|
||||
|
||||
@@ -177,7 +177,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
|
||||
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
||||
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
|
||||
if err := validateRunFlagValues(args); err != nil {
|
||||
@@ -375,7 +375,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
|
||||
}
|
||||
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
||||
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: loadedConfigPath,
|
||||
WorkingDir: workingDir,
|
||||
})
|
||||
@@ -462,7 +462,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||
Warnings: referenceWarnings,
|
||||
Diagnostics: referenceDiagnostics,
|
||||
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
|
||||
ChunkPlans: chunkPlans,
|
||||
Checkpoints: checkpointRecorder,
|
||||
@@ -471,7 +471,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
Debug: debugRecorder,
|
||||
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
|
||||
})
|
||||
commandState.observeOutput(output)
|
||||
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if diagnosticErr == nil {
|
||||
commandState.observeOutput(output, diagnosticProjection)
|
||||
}
|
||||
if err != nil {
|
||||
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
|
||||
if output.Manifest.PipelineID != "" {
|
||||
@@ -479,15 +482,21 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
|
||||
}
|
||||
}
|
||||
if diagnosticErr != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
}
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
|
||||
}
|
||||
if diagnosticErr != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
}
|
||||
|
||||
if err := writePartialSummary(summary, output); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
|
||||
}
|
||||
var encodedResult []byte
|
||||
if *machineOutput {
|
||||
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
|
||||
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
@@ -513,12 +522,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
|
||||
}
|
||||
}
|
||||
if len(output.Warnings) > 0 {
|
||||
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
|
||||
if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 {
|
||||
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount)
|
||||
if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok {
|
||||
fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile))
|
||||
}
|
||||
fmt.Fprintln(stderr)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) {
|
||||
for _, file := range files {
|
||||
if file.Name == name {
|
||||
return file.Name, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
|
||||
if summary == nil {
|
||||
return nil
|
||||
@@ -538,7 +560,7 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := summary.WriteWarnings(output.Warnings); err != nil {
|
||||
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
|
||||
return err
|
||||
}
|
||||
return summary.WriteCheckpointEvents(output.CheckpointEvents)
|
||||
@@ -1265,11 +1287,21 @@ type cliReferenceUnbindRequest struct {
|
||||
}
|
||||
|
||||
type cliReferenceSelector struct {
|
||||
Scope cliReferenceSelectorScope
|
||||
LaneID string
|
||||
Stage pipeline.ModuleStage
|
||||
SlotName string
|
||||
}
|
||||
|
||||
type cliReferenceSelectorScope uint8
|
||||
|
||||
const (
|
||||
cliReferenceScopePipeline cliReferenceSelectorScope = iota
|
||||
cliReferenceScopeLane
|
||||
cliReferenceScopeChunk
|
||||
cliReferenceScopeBinding
|
||||
)
|
||||
|
||||
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
@@ -1278,7 +1310,7 @@ func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
||||
for _, raw := range values {
|
||||
name, source, ok := strings.Cut(raw, "=")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
|
||||
return nil, fmt.Errorf("--reference must use slot=path, lane.slot=path, or lane.stage.slot=path")
|
||||
}
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
||||
@@ -1328,17 +1360,14 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
|
||||
}
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopePipeline, SlotName: strings.TrimSpace(parts[0])}, nil
|
||||
case 2:
|
||||
first := strings.TrimSpace(parts[0])
|
||||
slotName := strings.TrimSpace(parts[1])
|
||||
if first == string(pipeline.StageChunk) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeChunk, Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||
}
|
||||
if first == string(pipeline.StageMerge) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
|
||||
}
|
||||
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeLane, LaneID: first, SlotName: slotName}, nil
|
||||
case 3:
|
||||
laneID := strings.TrimSpace(parts[0])
|
||||
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
|
||||
@@ -1346,9 +1375,9 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
|
||||
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeBinding, LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||
default:
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1369,37 +1398,153 @@ func resolveCLIReferenceRequests(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
||||
// Broad CLI selectors are only presentation syntax. Collapse them into one
|
||||
// highest-specificity action per concrete framework target before pipeline
|
||||
// resolution so the generic reference contract stays stage-and-lane exact.
|
||||
actions := make(map[cliReferenceTargetKey]resolvedCLIReferenceAction)
|
||||
for _, request := range referenceRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
matches, err := resolveCLIReferenceTargets(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
Source: request.Source,
|
||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||
})
|
||||
for _, target := range matches {
|
||||
candidate := resolvedCLIReferenceAction{
|
||||
kind: cliReferenceActionBind,
|
||||
selector: request.Selector,
|
||||
target: target,
|
||||
slotName: request.Selector.SlotName,
|
||||
source: request.Source,
|
||||
specificity: request.Selector.specificity(),
|
||||
}
|
||||
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
||||
for _, request := range unbindRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
matches, err := resolveCLIReferenceTargets(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
})
|
||||
for _, target := range matches {
|
||||
candidate := resolvedCLIReferenceAction{
|
||||
kind: cliReferenceActionUnbind,
|
||||
selector: request.Selector,
|
||||
target: target,
|
||||
slotName: request.Selector.SlotName,
|
||||
specificity: request.Selector.specificity(),
|
||||
}
|
||||
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolved := make([]resolvedCLIReferenceAction, 0, len(actions))
|
||||
for _, action := range actions {
|
||||
resolved = append(resolved, action)
|
||||
}
|
||||
sort.Slice(resolved, func(i, j int) bool {
|
||||
left, right := resolved[i], resolved[j]
|
||||
if left.target.laneID != right.target.laneID {
|
||||
return left.target.laneID < right.target.laneID
|
||||
}
|
||||
if left.target.stage != right.target.stage {
|
||||
return referenceStageOrder(left.target.stage) < referenceStageOrder(right.target.stage)
|
||||
}
|
||||
return left.slotName < right.slotName
|
||||
})
|
||||
|
||||
overrides := make([]pipeline.ReferenceBinding, 0, len(resolved))
|
||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(resolved))
|
||||
for _, action := range resolved {
|
||||
switch action.kind {
|
||||
case cliReferenceActionBind:
|
||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||
Stage: action.target.stage,
|
||||
LaneID: action.target.laneID,
|
||||
SlotName: action.slotName,
|
||||
Source: action.source,
|
||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||
})
|
||||
case cliReferenceActionUnbind:
|
||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||
Stage: action.target.stage,
|
||||
LaneID: action.target.laneID,
|
||||
SlotName: action.slotName,
|
||||
})
|
||||
}
|
||||
}
|
||||
return overrides, unbinds, nil
|
||||
}
|
||||
|
||||
type cliReferenceActionKind uint8
|
||||
|
||||
const (
|
||||
cliReferenceActionBind cliReferenceActionKind = iota
|
||||
cliReferenceActionUnbind
|
||||
)
|
||||
|
||||
type cliReferenceTargetKey struct {
|
||||
stage pipeline.ModuleStage
|
||||
laneID string
|
||||
slotName string
|
||||
}
|
||||
|
||||
type resolvedCLIReferenceAction struct {
|
||||
kind cliReferenceActionKind
|
||||
selector cliReferenceSelector
|
||||
target selectedReferenceTarget
|
||||
slotName string
|
||||
source string
|
||||
specificity int
|
||||
}
|
||||
|
||||
func mergeCLIReferenceAction(actions map[cliReferenceTargetKey]resolvedCLIReferenceAction, candidate resolvedCLIReferenceAction) error {
|
||||
key := cliReferenceTargetKey{stage: candidate.target.stage, laneID: candidate.target.laneID, slotName: candidate.slotName}
|
||||
current, ok := actions[key]
|
||||
if !ok || candidate.specificity > current.specificity {
|
||||
actions[key] = candidate
|
||||
return nil
|
||||
}
|
||||
if candidate.specificity < current.specificity {
|
||||
return nil
|
||||
}
|
||||
if candidate.kind != current.kind {
|
||||
return fmt.Errorf("reference target %q slot %q is both bound by %q and unbound by %q at the same specificity", targetLabel(candidate.target), candidate.slotName, current.selector.String(), candidate.selector.String())
|
||||
}
|
||||
actions[key] = candidate
|
||||
return nil
|
||||
}
|
||||
|
||||
func (selector cliReferenceSelector) specificity() int {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
return 0
|
||||
case cliReferenceScopeLane:
|
||||
return 1
|
||||
case cliReferenceScopeChunk, cliReferenceScopeBinding:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func (selector cliReferenceSelector) String() string {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
return selector.SlotName
|
||||
case cliReferenceScopeLane:
|
||||
return selector.LaneID + "." + selector.SlotName
|
||||
case cliReferenceScopeChunk:
|
||||
return "chunk." + selector.SlotName
|
||||
case cliReferenceScopeBinding:
|
||||
return selector.LaneID + "." + string(selector.Stage) + "." + selector.SlotName
|
||||
default:
|
||||
return selector.SlotName
|
||||
}
|
||||
}
|
||||
|
||||
type selectedReferenceTarget struct {
|
||||
laneID string
|
||||
stage pipeline.ModuleStage
|
||||
@@ -1616,114 +1761,77 @@ func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
|
||||
return slotSet
|
||||
}
|
||||
|
||||
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
|
||||
func resolveCLIReferenceTargets(targets []selectedReferenceTarget, selector cliReferenceSelector) ([]selectedReferenceTarget, error) {
|
||||
slotName := strings.TrimSpace(selector.SlotName)
|
||||
if slotName == "" {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
|
||||
return nil, fmt.Errorf("reference slot must not be empty")
|
||||
}
|
||||
if selector.Stage == pipeline.StageChunk {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
matches := make([]selectedReferenceTarget, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by any selected target", slotName)
|
||||
}
|
||||
return matches, nil
|
||||
case cliReferenceScopeChunk:
|
||||
for _, target := range targets {
|
||||
if target.stage != pipeline.StageChunk {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
|
||||
}
|
||||
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
|
||||
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
|
||||
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
|
||||
return []selectedReferenceTarget{target}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reference chunk target is not selected")
|
||||
case cliReferenceScopeLane:
|
||||
laneSelected := false
|
||||
matches := make([]selectedReferenceTarget, 0, 3)
|
||||
for _, target := range targets {
|
||||
if target.laneID == selector.LaneID && target.stage == selector.Stage {
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||
}
|
||||
return target, nil
|
||||
if target.laneID != selector.LaneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
if strings.TrimSpace(selector.LaneID) != "" {
|
||||
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
|
||||
}
|
||||
return resolveCLIReferenceFlatTarget(targets, slotName)
|
||||
}
|
||||
|
||||
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.stage != stage {
|
||||
continue
|
||||
if !laneSelected {
|
||||
return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, selector.LaneID)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
return matches, nil
|
||||
case cliReferenceScopeBinding:
|
||||
laneSelected := false
|
||||
for _, target := range targets {
|
||||
if target.laneID != selector.LaneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if target.stage != selector.Stage {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||
}
|
||||
return []selectedReferenceTarget{target}, nil
|
||||
}
|
||||
if !laneSelected {
|
||||
return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
return nil, fmt.Errorf("reference %s target is not selected for lane %q", selector.Stage, selector.LaneID)
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
return nil, fmt.Errorf("reference selector has unknown scope")
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
|
||||
laneSelected := false
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.laneID != laneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
if !laneSelected {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func targetList(targets []selectedReferenceTarget) string {
|
||||
labels := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
labels = append(labels, targetLabel(target))
|
||||
}
|
||||
sort.Strings(labels)
|
||||
return strings.Join(labels, ", ")
|
||||
}
|
||||
|
||||
func targetLabel(target selectedReferenceTarget) string {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
return "chunk"
|
||||
@@ -1731,17 +1839,19 @@ func targetLabel(target selectedReferenceTarget) string {
|
||||
return target.laneID + "." + string(target.stage)
|
||||
}
|
||||
|
||||
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
|
||||
suggestions := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
suggestions = append(suggestions, "chunk."+slotName)
|
||||
continue
|
||||
}
|
||||
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
|
||||
func referenceStageOrder(stage pipeline.ModuleStage) int {
|
||||
switch stage {
|
||||
case pipeline.StageChunk:
|
||||
return 0
|
||||
case pipeline.StageExtract:
|
||||
return 1
|
||||
case pipeline.StageMerge:
|
||||
return 2
|
||||
case pipeline.StageNormalize:
|
||||
return 3
|
||||
default:
|
||||
return 4
|
||||
}
|
||||
sort.Strings(suggestions)
|
||||
return strings.Join(suggestions, " or ")
|
||||
}
|
||||
|
||||
func sortedPipelineIDs(cfg config.Config) []string {
|
||||
|
||||
@@ -590,10 +590,11 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.includeWarnings = true
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
||||
harness.includeWarningFile = true
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
||||
@@ -601,11 +602,12 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
||||
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
||||
t.Fatalf("durable output = %q, %v", output, err)
|
||||
}
|
||||
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
var warnings []contracts.Warning
|
||||
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
|
||||
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
|
||||
t.Fatalf("debug warnings = %#v", warnings)
|
||||
var diagnostics contracts.DiagnosticCollection
|
||||
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
|
||||
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
|
||||
t.Fatalf("debug diagnostics = %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,25 +7,40 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const runResultSchemaVersion = "notarius.run-result.v1"
|
||||
const runResultSchemaVersion = "notarius.run-result.v2"
|
||||
|
||||
type runResult struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file,omitempty"`
|
||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
||||
RejectedOutputCount int `json:"rejected_output_count"`
|
||||
WarningCount int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file,omitempty"`
|
||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
||||
RejectedOutputCount int `json:"rejected_output_count"`
|
||||
WarningGroupCount int `json:"warning_group_count"`
|
||||
WarningOccurrenceCount int `json:"warning_occurrence_count"`
|
||||
DiagnosticGroupCount int `json:"diagnostic_group_count"`
|
||||
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
|
||||
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||
}
|
||||
|
||||
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
||||
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
|
||||
}
|
||||
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
|
||||
}
|
||||
|
||||
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
|
||||
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a run ID")
|
||||
}
|
||||
@@ -51,14 +66,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
||||
}
|
||||
|
||||
result := runResult{
|
||||
SchemaVersion: runResultSchemaVersion,
|
||||
RunID: output.Manifest.RunID,
|
||||
PipelineID: output.Manifest.PipelineID,
|
||||
OutputDirectory: absOutputDirectory,
|
||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
||||
RejectedOutputCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
SchemaVersion: runResultSchemaVersion,
|
||||
RunID: output.Manifest.RunID,
|
||||
PipelineID: output.Manifest.PipelineID,
|
||||
OutputDirectory: absOutputDirectory,
|
||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
||||
RejectedOutputCount: len(output.Rejected),
|
||||
WarningGroupCount: len(diagnosticProjection.Warnings),
|
||||
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
|
||||
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
|
||||
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
|
||||
DiagnosticsTruncated: output.Diagnostics.Truncated,
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
|
||||
}
|
||||
|
||||
if strings.TrimSpace(debugDirectory) != "" {
|
||||
@@ -85,6 +105,17 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
||||
if len(summaries) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
||||
for index, summary := range summaries {
|
||||
cloned[index] = artifacts.CloneValidationSummary(summary)
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func encodeRunResult(result runResult) ([]byte, error) {
|
||||
encoded, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := receipt["run_id"]; got != productionRunID {
|
||||
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["warning_count"]; got != float64(0) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "approved" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
@@ -63,19 +63,19 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["warning_count"]; got != float64(1) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||
@@ -92,7 +92,7 @@ func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -52,8 +52,11 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["warning_count"]; got != float64(1) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", decoded)
|
||||
}
|
||||
if got := decoded["validation_summaries"]; got != nil {
|
||||
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
||||
}
|
||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||
t.Fatalf("output_directory = %q", got)
|
||||
@@ -112,6 +115,26 @@ func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
||||
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
||||
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
||||
t.Fatalf("encoded result = %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -159,8 +182,15 @@ func testRunOutput() pipeline.RunOutput {
|
||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
||||
Rejected: []contracts.RejectedOutput{{}},
|
||||
Warnings: []contracts.Warning{{}},
|
||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
|
||||
Disposition: contracts.DiagnosticDispositionWarning,
|
||||
Category: contracts.DiagnosticCategoryFallback,
|
||||
ReasonCode: "fallback",
|
||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
|
||||
}}},
|
||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -33,13 +34,17 @@ func (s *pipelineCommandState) setDebugPath(debugPath string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
|
||||
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput, diagnostics contracts.DiagnosticProjection) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.report.OutputCount = len(output.NormalizeOutputs)
|
||||
s.report.RejectedCount = len(output.Rejected)
|
||||
s.report.WarningCount = len(output.Warnings)
|
||||
s.report.WarningGroupCount = len(diagnostics.Warnings)
|
||||
s.report.WarningOccurrenceCount = diagnostics.WarningOccurrenceCount
|
||||
s.report.DiagnosticGroupCount = len(diagnostics.Diagnostics)
|
||||
s.report.DiagnosticOccurrenceCount = diagnostics.DiagnosticOccurrenceCount
|
||||
s.report.DiagnosticsTruncated = output.Diagnostics.Truncated
|
||||
s.report.ValidationStatus = output.Manifest.ValidationStatus
|
||||
}
|
||||
|
||||
|
||||
@@ -238,7 +238,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
||||
},
|
||||
}
|
||||
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
|
||||
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -264,7 +264,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
||||
}
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
|
||||
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
|
||||
@@ -19,12 +19,12 @@ import (
|
||||
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
const retries = 2
|
||||
tests := []struct {
|
||||
name string
|
||||
responses []string
|
||||
wantCalls int
|
||||
wantRejected bool
|
||||
wantSpell string
|
||||
wantWarningCode string
|
||||
name string
|
||||
responses []string
|
||||
wantCalls int
|
||||
wantRejected bool
|
||||
wantSpell string
|
||||
wantAdvisoryCode string
|
||||
}{
|
||||
{
|
||||
name: "unknown spell remains rejected after exhaustion",
|
||||
@@ -42,9 +42,9 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Aegis of Emberfall"),
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantSpell: "Aegis of Emberfall",
|
||||
wantWarningCode: "spell_not_near_source",
|
||||
wantCalls: 2,
|
||||
wantSpell: "Aegis of Emberfall",
|
||||
wantAdvisoryCode: "spell_not_near_source",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -65,6 +65,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
t.Fatalf("materialize production references: %v", err)
|
||||
}
|
||||
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
||||
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
|
||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
@@ -91,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want no emitted warnings from rejected attempts", output.Warnings)
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
|
||||
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -107,8 +108,16 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||
}
|
||||
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode {
|
||||
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings)
|
||||
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
|
||||
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 2 {
|
||||
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
|
||||
}
|
||||
for _, diagnostic := range output.Diagnostics.Groups {
|
||||
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
|
||||
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -349,7 +349,7 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
|
||||
t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID)
|
||||
}
|
||||
report := readStateTestRunReport(t, debugPath)
|
||||
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
|
||||
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "approved" {
|
||||
t.Fatalf("success report = %#v", report)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
|
||||
@@ -431,7 +431,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
|
||||
bundlePath := onlyChildDir(t, roots.debug)
|
||||
runID := filepath.Base(bundlePath)
|
||||
report := readStateTestRunReport(t, bundlePath)
|
||||
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
|
||||
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != tc.wantValidation {
|
||||
t.Fatalf("failure report = %#v", report)
|
||||
}
|
||||
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
||||
@@ -445,7 +445,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
|
||||
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")}
|
||||
harness.extractErr = errors.New("synthetic partial pipeline failure")
|
||||
|
||||
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
|
||||
@@ -454,7 +454,7 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
||||
}
|
||||
bundlePath := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, bundlePath)
|
||||
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
|
||||
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningGroupCount != 1 || report.WarningOccurrenceCount != 1 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "failed" {
|
||||
t.Fatalf("partial failure report = %#v", report)
|
||||
}
|
||||
|
||||
@@ -463,10 +463,10 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
||||
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
|
||||
t.Fatalf("partial manifest = %#v", manifest)
|
||||
}
|
||||
var warnings []contracts.Warning
|
||||
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
|
||||
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
|
||||
t.Fatalf("partial warnings = %#v", warnings)
|
||||
var diagnostics contracts.DiagnosticCollection
|
||||
readStateTestSummaryJSON(t, bundlePath, "final-diagnostics.json", &diagnostics)
|
||||
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" {
|
||||
t.Fatalf("partial diagnostics = %#v", diagnostics)
|
||||
}
|
||||
var events []pipeline.CheckpointEvent
|
||||
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
|
||||
@@ -863,11 +863,12 @@ type stateTestHarness struct {
|
||||
chunkCalls, extractCalls int
|
||||
runIDCalls uint64
|
||||
extractErr error
|
||||
chunkWarnings []contracts.Warning
|
||||
chunkDiagnostics []contracts.ProducerDiagnostic
|
||||
moduleProfiles []string
|
||||
sessionIDs []string
|
||||
outputWarnings []contracts.Warning
|
||||
outputDiagnostics contracts.DiagnosticCollection
|
||||
includeWarnings bool
|
||||
includeWarningFile bool
|
||||
}
|
||||
|
||||
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
||||
@@ -892,7 +893,7 @@ func (h *stateTestHarness) options() Options {
|
||||
panic(err)
|
||||
}
|
||||
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
|
||||
return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -931,7 +932,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
|
||||
c.harness.mu.Lock()
|
||||
c.harness.chunkCalls++
|
||||
c.harness.mu.Unlock()
|
||||
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
|
||||
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil
|
||||
}
|
||||
|
||||
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
|
||||
@@ -999,20 +1000,28 @@ func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNor
|
||||
}
|
||||
|
||||
type stateTestOutput struct {
|
||||
harness *stateTestHarness
|
||||
includeWarnings bool
|
||||
harness *stateTestHarness
|
||||
includeDiagnostics bool
|
||||
}
|
||||
|
||||
func (o stateTestOutput) Key() string { return "test/output" }
|
||||
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
o.harness.mu.Lock()
|
||||
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
|
||||
o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics)
|
||||
o.harness.mu.Unlock()
|
||||
data := []byte("{\"ok\":true}\n")
|
||||
if o.includeWarnings && len(req.Warnings) > 0 {
|
||||
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
|
||||
if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
|
||||
data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
|
||||
}
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
|
||||
files := []contracts.OutputFile{{Name: "result.json", Bytes: data}}
|
||||
if o.harness.includeWarningFile {
|
||||
files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")})
|
||||
}
|
||||
return contracts.OutputResult{Files: files}, nil
|
||||
}
|
||||
|
||||
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {
|
||||
return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}}
|
||||
}
|
||||
|
||||
type failingDebugRecorder struct{}
|
||||
|
||||
@@ -93,17 +93,44 @@ type NormalizedOutputManifest struct {
|
||||
}
|
||||
|
||||
type RejectedOutputManifest struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Validation *ValidationSummary `json:"validation,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationSummary is the bounded, durable outcome of validating one
|
||||
// producer result. It deliberately contains identities and stable codes, not
|
||||
// model responses, corrective guidance, validator diagnostics, or payloads.
|
||||
type ValidationSummary struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
Status string `json:"status"`
|
||||
RejectingValidators []string `json:"rejecting_validators,omitempty"`
|
||||
ReasonCodes []string `json:"reason_codes,omitempty"`
|
||||
IncompleteValidators []string `json:"incomplete_validators,omitempty"`
|
||||
ProducerAttemptCount int `json:"producer_attempt_count"`
|
||||
TerminalAction string `json:"terminal_action"`
|
||||
}
|
||||
|
||||
// CloneValidationSummary returns an independently owned durable summary.
|
||||
func CloneValidationSummary(summary ValidationSummary) ValidationSummary {
|
||||
summary.RejectingValidators = append([]string(nil), summary.RejectingValidators...)
|
||||
summary.ReasonCodes = append([]string(nil), summary.ReasonCodes...)
|
||||
summary.IncompleteValidators = append([]string(nil), summary.IncompleteValidators...)
|
||||
return summary
|
||||
}
|
||||
|
||||
type CheckpointDecisionManifest struct {
|
||||
@@ -163,6 +190,7 @@ type RunManifest struct {
|
||||
References []ReferenceProvenance `json:"references,omitempty"`
|
||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||
ValidationSummaries []ValidationSummary `json:"validation_summaries,omitempty"`
|
||||
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
|
||||
@@ -116,6 +116,7 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
|
||||
|
||||
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||
out := in
|
||||
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||
if in.StructuredOutputRepairAttempts != nil {
|
||||
value := *in.StructuredOutputRepairAttempts
|
||||
out.StructuredOutputRepairAttempts = &value
|
||||
@@ -200,6 +201,7 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
|
||||
|
||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||
out := in
|
||||
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
|
||||
if in.StructuredOutputRepairAttempts != nil {
|
||||
value := *in.StructuredOutputRepairAttempts
|
||||
out.StructuredOutputRepairAttempts = &value
|
||||
@@ -212,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := *in
|
||||
if in.ProducerStructuralFailure != nil {
|
||||
value := *in.ProducerStructuralFailure
|
||||
out.ProducerStructuralFailure = &value
|
||||
}
|
||||
if in.SemanticRejection != nil {
|
||||
value := *in.SemanticRejection
|
||||
out.SemanticRejection = &value
|
||||
}
|
||||
if in.ValidatorFailure != nil {
|
||||
value := *in.ValidatorFailure
|
||||
out.ValidatorFailure = &value
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||
if len(in.Validators) > 0 {
|
||||
|
||||
@@ -37,6 +37,7 @@ type FilePromptKitLocalBackendConfig struct {
|
||||
type FilePipelineProfile struct {
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
@@ -55,12 +56,19 @@ func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||
type plainFilePipelineProfile FilePipelineProfile
|
||||
var decoded plainFilePipelineProfile
|
||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||
"llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||
}, "pipeline profile")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*p = FilePipelineProfile(decoded)
|
||||
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
|
||||
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.ValidationPolicy = policy
|
||||
}
|
||||
_, p.artifactsSet = seen["artifacts"]
|
||||
_, p.stepsSet = seen["steps"]
|
||||
_, p.llmProfileSet = seen["llm_profile"]
|
||||
@@ -154,6 +162,7 @@ type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
ValidationPolicy *pipeline.ValidationPolicyOverride
|
||||
Retries int
|
||||
Options map[string]any
|
||||
References map[string]fileReferenceSource
|
||||
@@ -250,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
b.Module = strings.TrimSpace(module)
|
||||
return nil
|
||||
case yaml.MappingNode:
|
||||
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
keyNode := node.Content[i]
|
||||
valueNode := node.Content[i+1]
|
||||
if _, exists := seen[keyNode.Value]; exists {
|
||||
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
|
||||
}
|
||||
seen[keyNode.Value] = struct{}{}
|
||||
switch keyNode.Value {
|
||||
case "module":
|
||||
var module string
|
||||
@@ -275,6 +289,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
return err
|
||||
}
|
||||
b.StructuredOutputRepairAttempts = attempts
|
||||
case "validation_policy":
|
||||
policy, err := parseValidationPolicy(valueNode, "module binding")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.ValidationPolicy = policy
|
||||
case "retries":
|
||||
var retries int
|
||||
if err := valueNode.Decode(&retries); err != nil {
|
||||
@@ -318,6 +338,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
Module: strings.TrimSpace(b.Module),
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
|
||||
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
|
||||
Retries: b.Retries,
|
||||
Options: cloneOptions(b.Options),
|
||||
References: fileReferenceSourcesToPipeline(b.References),
|
||||
@@ -325,6 +346,51 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
}
|
||||
}
|
||||
|
||||
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value == key {
|
||||
return node.Content[i+1], true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
|
||||
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
|
||||
return nil, fmt.Errorf("%s validation_policy must be an object", context)
|
||||
}
|
||||
policy := &pipeline.ValidationPolicyOverride{}
|
||||
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i].Value
|
||||
value := node.Content[i+1]
|
||||
if _, exists := seen[key]; exists {
|
||||
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
|
||||
}
|
||||
switch key {
|
||||
case "producer_structural_failure":
|
||||
value := pipeline.ProducerStructuralFailureAction(value.Value)
|
||||
policy.ProducerStructuralFailure = &value
|
||||
case "semantic_rejection":
|
||||
value := pipeline.SemanticRejectionAction(value.Value)
|
||||
policy.SemanticRejection = &value
|
||||
case "validator_failure":
|
||||
value := pipeline.ValidatorFailureAction(value.Value)
|
||||
policy.ValidatorFailure = &value
|
||||
default:
|
||||
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
|
||||
}
|
||||
}
|
||||
if err := policy.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("%s must be an object", context)
|
||||
@@ -573,6 +639,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
ID: pipelineID,
|
||||
LLMProfile: llmProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
|
||||
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||
|
||||
@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
|
||||
|
||||
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
|
||||
out := in
|
||||
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
||||
out.Input = redactBinding(cloneModuleBinding(in.Input))
|
||||
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
|
||||
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||
|
||||
@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
|
||||
semantic := pipeline.SemanticRejectionRejectOutput
|
||||
validator := pipeline.ValidatorFailureFailRun
|
||||
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
|
||||
effective := EffectiveConfig{
|
||||
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
|
||||
"main": {ValidationPolicy: configured},
|
||||
}},
|
||||
ResolvedPipeline: pipeline.ResolvedPipeline{
|
||||
ConfiguredValidationPolicy: configured,
|
||||
ChunkValidationPolicy: pipeline.ValidationPolicy{
|
||||
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: pipeline.ValidatorFailureFailRun,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
|
||||
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
|
||||
}
|
||||
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
|
||||
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
|
||||
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
|
||||
t.Fatal("redacted payload aliases validation policy")
|
||||
}
|
||||
}
|
||||
|
||||
func redactionTestBinding(name string) pipeline.ModuleBinding {
|
||||
return pipeline.ModuleBinding{
|
||||
Module: "safe-" + name,
|
||||
|
||||
@@ -119,6 +119,11 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
||||
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
if profile.ValidationPolicy != nil {
|
||||
if err := profile.ValidationPolicy.Validate(); err != nil {
|
||||
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
|
||||
}
|
||||
}
|
||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -198,6 +203,16 @@ func validateBinding(
|
||||
binding pipeline.ModuleBinding,
|
||||
referencesAllowed bool,
|
||||
) error {
|
||||
if binding.ValidationPolicy != nil {
|
||||
switch slot {
|
||||
case "chunk", "extract", "merge", "normalize":
|
||||
default:
|
||||
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
|
||||
}
|
||||
if err := binding.ValidationPolicy.Validate(); err != nil {
|
||||
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
|
||||
}
|
||||
}
|
||||
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -255,8 +270,11 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
|
||||
if validator.Validators.Set {
|
||||
return fmt.Errorf("%s nested validators are not supported", context)
|
||||
}
|
||||
if validator.Retries != 0 {
|
||||
return fmt.Errorf("%s retries are not supported", context)
|
||||
if validator.ValidationPolicy != nil {
|
||||
return fmt.Errorf("%s validation_policy is not supported", context)
|
||||
}
|
||||
if validator.Retries < 0 {
|
||||
return fmt.Errorf("%s retries must be greater than or equal to zero", context)
|
||||
}
|
||||
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
||||
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
||||
|
||||
@@ -374,17 +374,17 @@ func TestValidateValidatorBindingRules(t *testing.T) {
|
||||
want: "chunk validators[0] module must not be empty",
|
||||
},
|
||||
{
|
||||
name: "validator retries",
|
||||
name: "negative validator retries",
|
||||
setup: func(profile *pipeline.PipelineProfile) {
|
||||
profile.Chunk.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{{
|
||||
Module: "validator",
|
||||
Retries: 1,
|
||||
Retries: -1,
|
||||
}},
|
||||
}
|
||||
},
|
||||
want: "chunk validators[0] retries are not supported",
|
||||
want: "chunk validators[0] retries must be greater than or equal to zero",
|
||||
},
|
||||
{
|
||||
name: "validator references",
|
||||
|
||||
97
internal/core/config/validation_policy_contract_test.go
Normal file
97
internal/core/config/validation_policy_contract_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
|
||||
const valid = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
validation_policy:
|
||||
producer_structural_failure: reject_output
|
||||
semantic_rejection: fail_run
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
validation_policy:
|
||||
validator_failure: fail_run
|
||||
artifacts:
|
||||
lane:
|
||||
extract:
|
||||
module: extract
|
||||
validation_policy:
|
||||
semantic_rejection: reject_output
|
||||
`
|
||||
cfg := applyFileConfig(t, valid)
|
||||
profile := cfg.Pipelines["main"]
|
||||
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
|
||||
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
|
||||
}
|
||||
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
|
||||
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
|
||||
}
|
||||
lane := profile.Artifacts["lane"]
|
||||
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
|
||||
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
yaml string
|
||||
}{
|
||||
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
|
||||
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
|
||||
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
|
||||
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
|
||||
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
|
||||
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationPolicyPlacementRules(t *testing.T) {
|
||||
policy := &pipeline.ValidationPolicyOverride{}
|
||||
semantic := pipeline.SemanticRejectionRejectOutput
|
||||
policy.SemanticRejection = &semantic
|
||||
base := pipeline.PipelineProfile{
|
||||
ID: "main",
|
||||
Input: pipeline.Binding("input"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"lane": {Extract: pipeline.Binding("extract")},
|
||||
},
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*pipeline.PipelineProfile)
|
||||
}{
|
||||
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
|
||||
{"output", func(profile *pipeline.PipelineProfile) {
|
||||
profile.Output = pipeline.Binding("output")
|
||||
profile.Output.ValidationPolicy = policy
|
||||
}},
|
||||
{"validator", func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["lane"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
|
||||
profile.Artifacts["lane"] = lane
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := base
|
||||
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
|
||||
test.mutate(&profile)
|
||||
cfg := Default()
|
||||
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
|
||||
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
|
||||
if err := summary.WriteRunReport(RunReport{RunID: bundle.RunID(), PipelineID: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := summary.WriteWarnings([]contracts.Warning{{ReasonCode: "test"}}); err != nil {
|
||||
if err := summary.WriteDiagnostics(contracts.DiagnosticCollection{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := summary.WriteError("failed"); err != nil {
|
||||
@@ -126,7 +126,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
|
||||
ArtifactRunManifest,
|
||||
ArtifactChunkPlan,
|
||||
ArtifactRunReport,
|
||||
ArtifactWarnings,
|
||||
ArtifactDiagnostics,
|
||||
ArtifactErrorLog,
|
||||
} {
|
||||
info, err := os.Stat(filepath.Join(bundle.SummaryRoot(), name))
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
ArtifactRunManifest = "run-manifest.json"
|
||||
ArtifactChunkPlan = "chunk-plan.json"
|
||||
ArtifactRunReport = "run-report.json"
|
||||
ArtifactWarnings = "warnings.json"
|
||||
ArtifactDiagnostics = "final-diagnostics.json"
|
||||
ArtifactErrorLog = "error.log"
|
||||
)
|
||||
|
||||
@@ -45,15 +45,19 @@ type Invocation struct {
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
type RunReport struct {
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
DebugPath string `json:"debug_path,omitempty"`
|
||||
Succeeded bool `json:"succeeded"`
|
||||
OutputCount int `json:"output_count"`
|
||||
RejectedCount int `json:"rejected_count"`
|
||||
WarningCount int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
DebugPath string `json:"debug_path,omitempty"`
|
||||
Succeeded bool `json:"succeeded"`
|
||||
OutputCount int `json:"output_count"`
|
||||
RejectedCount int `json:"rejected_count"`
|
||||
WarningGroupCount int `json:"warning_group_count"`
|
||||
WarningOccurrenceCount int `json:"warning_occurrence_count"`
|
||||
DiagnosticGroupCount int `json:"diagnostic_group_count"`
|
||||
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
|
||||
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
}
|
||||
type SummaryWriter struct {
|
||||
root, runID string
|
||||
@@ -101,8 +105,8 @@ func (w *SummaryWriter) WriteChunkPlan(v artifacts.ChunkPlanSummary) error {
|
||||
return w.writeJSON(ArtifactChunkPlan, v)
|
||||
}
|
||||
func (w *SummaryWriter) WriteRunReport(v RunReport) error { return w.writeJSON(ArtifactRunReport, v) }
|
||||
func (w *SummaryWriter) WriteWarnings(v []contracts.Warning) error {
|
||||
return w.writeJSON(ArtifactWarnings, v)
|
||||
func (w *SummaryWriter) WriteDiagnostics(v contracts.DiagnosticCollection) error {
|
||||
return w.writeJSON(ArtifactDiagnostics, contracts.CloneDiagnosticCollection(v))
|
||||
}
|
||||
func (w *SummaryWriter) WriteError(message string) error {
|
||||
return w.writeBytes(ArtifactErrorLog, []byte(message+"\n"))
|
||||
|
||||
@@ -24,7 +24,6 @@ type filesystemCheckpointFixture struct {
|
||||
merge pipeline.CheckpointArtifact
|
||||
normalize pipeline.CheckpointArtifact
|
||||
dependencies []pipeline.CheckpointFingerprint
|
||||
warnings []contracts.Warning
|
||||
rejected []contracts.RejectedOutput
|
||||
}
|
||||
|
||||
@@ -36,9 +35,9 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
|
||||
fixture.doc.Metadata["owner"] = "caller mutation"
|
||||
fixture.extract.Artifact.Content[0] = 'x'
|
||||
fixture.extract.Artifact.Metadata["content"] = "caller mutation"
|
||||
fixture.extract.Diagnostics[0].Diagnostic.Samples[0].Message = "caller mutation"
|
||||
fixture.merge.Artifact.Content[0] = 'x'
|
||||
fixture.normalize.Artifact.Content[0] = 'x'
|
||||
fixture.warnings[0].Message = "caller mutation"
|
||||
fixture.rejected[0].Message = "caller mutation"
|
||||
|
||||
t.Run("source", func(t *testing.T) {
|
||||
@@ -60,11 +59,11 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
got, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Warnings) != 1 {
|
||||
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Outputs[0].Diagnostics) != 1 {
|
||||
t.Fatalf("extract result=%#v decision=%#v", got, decision)
|
||||
}
|
||||
output := got.Outputs[0]
|
||||
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || got.Warnings[0].ReasonCode != "partial" || got.Rejected[0].ReasonCode != "invalid_source" {
|
||||
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || output.Diagnostics[0].Diagnostic.ReasonCode != "normalized_record" || got.Rejected[0].ReasonCode != "invalid_source" {
|
||||
t.Fatalf("extract values were not restored: %#v", got)
|
||||
}
|
||||
manifest := readManifest[ExtractLaneManifest](t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "manifest.json"))
|
||||
@@ -73,37 +72,36 @@ func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
|
||||
}
|
||||
|
||||
got.Outputs[0].Artifact.Content[0] = 'y'
|
||||
got.Warnings[0].Message = "loaded mutation"
|
||||
reloaded, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) || reloaded.Warnings[0].Message != "partial output" {
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) {
|
||||
t.Fatalf("extract reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
load func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision)
|
||||
load func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision)
|
||||
want []byte
|
||||
}{
|
||||
{name: "merge", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||
{name: "merge", load: func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision) {
|
||||
got, decision := fixture.loader.Merge("lane-a", "merge-module", fixture.dependencies)
|
||||
return got.Output, got.Warnings, decision
|
||||
return got.Output, decision
|
||||
}, want: []byte(`{"spells":["fire"]}`)},
|
||||
{name: "normalize", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||
{name: "normalize", load: func() (pipeline.CheckpointArtifact, pipeline.CheckpointDecision) {
|
||||
got, decision := fixture.loader.Normalize("lane-a", "normalize-module", fixture.dependencies)
|
||||
return got.Output, got.Warnings, decision
|
||||
return got.Output, decision
|
||||
}, want: []byte(`{"spells":["fire"],"normalized":true}`)},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, warnings, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(warnings) != 1 || warnings[0].ReasonCode != "review" {
|
||||
t.Fatalf("%s result=%#v warnings=%#v decision=%#v", tt.name, got, warnings, decision)
|
||||
got, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(got.Diagnostics) != 1 || got.Diagnostics[0].Diagnostic.ReasonCode != "normalized_record" {
|
||||
t.Fatalf("%s result=%#v decision=%#v", tt.name, got, decision)
|
||||
}
|
||||
|
||||
got.Artifact.Content[0] = 'z'
|
||||
reloaded, warnings, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) || warnings[0].Message != "review manually" {
|
||||
t.Fatalf("%s reload changed after loaded mutation: %#v warnings=%#v decision=%#v", tt.name, reloaded, warnings, decision)
|
||||
reloaded, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) {
|
||||
t.Fatalf("%s reload changed after loaded mutation: %#v decision=%#v", tt.name, reloaded, decision)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -159,6 +157,7 @@ func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
|
||||
}{
|
||||
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
|
||||
{"v2 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV2 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
|
||||
{"v3 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV3 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
|
||||
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
|
||||
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, pipeline.CheckpointReasonIdentityMismatch},
|
||||
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, pipeline.CheckpointReasonStageMismatch},
|
||||
@@ -204,6 +203,11 @@ func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T)
|
||||
{"content digest", func(m map[string]any) {
|
||||
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other"
|
||||
}, pipeline.CheckpointReasonArtifactDigestMismatch},
|
||||
{"diagnostics", func(m map[string]any) {
|
||||
m["outputs"].([]any)[0].(map[string]any)["diagnostics"] = []any{map[string]any{
|
||||
"diagnostic": map[string]any{"disposition": "warning", "category": "configuration", "reason_code": "invalid", "occurrence_count": float64(0)},
|
||||
}}
|
||||
}, pipeline.CheckpointReasonArtifactPayloadInvalid},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
@@ -337,9 +341,6 @@ func TestFilesystemLoaderReadsAcceptedNormalizeWithoutStageDependencies(t *testi
|
||||
if checkpoint.Output.Artifact.Content == nil || string(checkpoint.Output.Artifact.Content) != string(fixture.normalize.Artifact.Content) {
|
||||
t.Fatalf("accepted normalize output = %#v, want recorded artifact", checkpoint.Output)
|
||||
}
|
||||
if len(checkpoint.Warnings) != 1 || checkpoint.Warnings[0].ReasonCode != "normalized" {
|
||||
t.Fatalf("accepted normalize warnings = %#v", checkpoint.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemLoaderRejectsInvalidAcceptedNormalize(t *testing.T) {
|
||||
@@ -402,8 +403,7 @@ func seedAcceptedNormalizeCheckpoint(t *testing.T) filesystemCheckpointFixture {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stepRecorder := recorder.(pipeline.StepCheckpointRecorder)
|
||||
warnings := []contracts.Warning{{Scope: "normalize", ReasonCode: "normalized", Message: "normalized warning"}}
|
||||
if err := stepRecorder.NormalizeSucceededForStep("step-1", "lane-a", "normalize-module", []pipeline.CheckpointFingerprint{{Name: "merge", Value: "sha256:unavailable"}}, artifact, warnings); err != nil {
|
||||
if err := stepRecorder.NormalizeSucceededForStep("step-1", "lane-a", "normalize-module", []pipeline.CheckpointFingerprint{{Name: "merge", Value: "sha256:unavailable"}}, artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loader, err := NewFilesystemLoader(root, identity)
|
||||
@@ -493,20 +493,18 @@ func seedFilesystemCheckpoints(t *testing.T) filesystemCheckpointFixture {
|
||||
merge: checkpointArtifact("merge", `{"spells":["fire"]}`),
|
||||
normalize: checkpointArtifact("normalize", `{"spells":["fire"],"normalized":true}`),
|
||||
dependencies: []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:source"}, {Name: "chunk-plan", Value: "sha256:plan"}},
|
||||
warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "partial", Message: "partial output"}},
|
||||
rejected: []contracts.RejectedOutput{{Stage: "extract", LaneID: "lane-a", ModuleKey: "extract-module", ChunkID: "chunk-a", ValidatorName: "source_refs", ReasonCode: "invalid_source", Message: "source reference is invalid", AttemptCount: 1}},
|
||||
}
|
||||
if err := recorder.SourceSucceeded("source-module", &fixture.doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected, fixture.warnings); err != nil {
|
||||
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mergeWarnings := []contracts.Warning{{Scope: "merge", ReasonCode: "review", Message: "review manually"}}
|
||||
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge, mergeWarnings); err != nil {
|
||||
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize, mergeWarnings); err != nil {
|
||||
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture.loader, err = NewFilesystemLoader(root, identity)
|
||||
@@ -527,7 +525,8 @@ func checkpointArtifact(module, content string) pipeline.CheckpointArtifact {
|
||||
return pipeline.CheckpointArtifact{
|
||||
LaneID: "lane-a", ModuleKey: module, SourceID: "document-1", ChunkID: "chunk-a", ChunkIndex: 0,
|
||||
ChunkRef: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
|
||||
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
|
||||
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
|
||||
Diagnostics: []pipeline.CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionObservation, Category: contracts.DiagnosticCategoryNormalization, ReasonCode: "normalized_record", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fixture", Message: "record normalized"}}}}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, depe
|
||||
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
|
||||
return pipeline.ExtractCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
|
||||
}
|
||||
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
|
||||
@@ -112,7 +112,7 @@ func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, depend
|
||||
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
|
||||
return pipeline.MergeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
|
||||
}
|
||||
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
return pipeline.MergeCheckpoint{Output: values[0]}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *FilesystemLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
|
||||
@@ -138,7 +138,7 @@ func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, de
|
||||
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
|
||||
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
|
||||
}
|
||||
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
return pipeline.NormalizeCheckpoint{Output: values[0]}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *FilesystemLoader) AcceptedNormalize(stepID, laneID, moduleKey string) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
|
||||
@@ -163,7 +163,7 @@ func (l *FilesystemLoader) AcceptedNormalize(stepID, laneID, moduleKey string) (
|
||||
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
|
||||
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch)
|
||||
}
|
||||
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused)
|
||||
return pipeline.NormalizeCheckpoint{Output: values[0]}, decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused)
|
||||
}
|
||||
|
||||
func (l *FilesystemLoader) validateAcceptedNormalizeManifest(manifest StageManifest, stepID, laneID, moduleKey string) pipeline.CheckpointDecision {
|
||||
@@ -205,11 +205,33 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
|
||||
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
|
||||
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactCodecIncompatible, err: fmt.Errorf("artifact codec identity is incomplete")}
|
||||
}
|
||||
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
|
||||
diagnostics, err := cloneAndValidateCheckpointDiagnostics(v.Diagnostics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}, Diagnostics: diagnostics})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cloneAndValidateCheckpointDiagnostics(values []pipeline.CheckpointDiagnostic) ([]pipeline.CheckpointDiagnostic, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
diagnostics := make([]contracts.ProducerDiagnostic, len(values))
|
||||
for index, value := range values {
|
||||
diagnostics[index] = value.Diagnostic
|
||||
}
|
||||
if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil {
|
||||
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactPayloadInvalid, err: fmt.Errorf("checkpoint diagnostics: %w", err)}
|
||||
}
|
||||
cloned := make([]pipeline.CheckpointDiagnostic, len(values))
|
||||
for index, value := range values {
|
||||
cloned[index] = pipeline.CheckpointDiagnostic{Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{value.Diagnostic})[0], ValidatorKey: value.ValidatorKey}
|
||||
}
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
|
||||
if !l.Enabled() {
|
||||
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLoadingDisabled)
|
||||
|
||||
@@ -3,7 +3,8 @@ package checkpoint
|
||||
import "time"
|
||||
|
||||
const (
|
||||
WorkspaceSchemaVersion = "notarius.workspace.v3"
|
||||
WorkspaceSchemaVersion = "notarius.workspace.v4"
|
||||
WorkspaceSchemaVersionV3 = "notarius.workspace.v3"
|
||||
WorkspaceSchemaVersionV2 = "notarius.workspace.v2"
|
||||
WorkspaceSchemaVersionV1 = "notarius.workspace.v1"
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -80,18 +81,18 @@ func (r *FilesystemRecorder) ExtractRunningForStep(stepID, laneID string, module
|
||||
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
return r.ExtractSucceededForStep("", laneID, moduleKey, dependencies, outputs, rejected, warnings)
|
||||
func (r *FilesystemRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput) error {
|
||||
return r.ExtractSucceededForStep("", laneID, moduleKey, dependencies, outputs, rejected)
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) ExtractSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
|
||||
func (r *FilesystemRecorder) ExtractSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput) error {
|
||||
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected)}
|
||||
if err := r.writePayload(lanePayloadPath("extract", stepID, laneID, "outputs.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(StageExtract, statusForRejected(rejected), stepID, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests(outputs))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, rejected)
|
||||
manifest.ValidationStatus = validationStatusString(rejected)
|
||||
manifest.Rejections = rejectionSummaries(rejected)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
|
||||
@@ -118,17 +119,17 @@ func (r *FilesystemRecorder) MergeRunningForStep(stepID, laneID string, moduleKe
|
||||
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
return r.MergeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
|
||||
func (r *FilesystemRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
|
||||
return r.MergeSucceededForStep("", laneID, moduleKey, dependencies, output)
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) MergeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
if err := r.writePayload(lanePayloadPath("merge", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
|
||||
func (r *FilesystemRecorder) MergeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
|
||||
if err := r.writePayload(lanePayloadPath("merge", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output)}); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(StageMerge, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.ValidationStatus = validationStatusString(nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
@@ -166,17 +167,17 @@ func (r *FilesystemRecorder) NormalizeRunningForStep(stepID, laneID string, modu
|
||||
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
return r.NormalizeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
|
||||
func (r *FilesystemRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
|
||||
return r.NormalizeSucceededForStep("", laneID, moduleKey, dependencies, output)
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) NormalizeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
if err := r.writePayload(lanePayloadPath("normalize", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
|
||||
func (r *FilesystemRecorder) NormalizeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact) error {
|
||||
if err := r.writePayload(lanePayloadPath("normalize", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output)}); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(StageNormalize, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.ValidationStatus = validationStatusString(nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
@@ -248,39 +249,37 @@ type sourceDocumentEnvelope struct {
|
||||
}
|
||||
|
||||
type binaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type artifactCheckpointEnvelope struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
|
||||
Kind contracts.ArtifactKind `json:"artifact_kind"`
|
||||
Schema contracts.ArtifactSchema `json:"schema"`
|
||||
SchemaDigest string `json:"schema_digest"`
|
||||
Content binaryEnvelope `json:"content"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
|
||||
Kind contracts.ArtifactKind `json:"artifact_kind"`
|
||||
Schema contracts.ArtifactSchema `json:"schema"`
|
||||
SchemaDigest string `json:"schema_digest"`
|
||||
Content binaryEnvelope `json:"content"`
|
||||
Diagnostics []pipeline.CheckpointDiagnostic `json:"diagnostics,omitempty"`
|
||||
}
|
||||
type artifactExtractEnvelope struct {
|
||||
Outputs []artifactCheckpointEnvelope `json:"outputs"`
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
type artifactSingleEnvelope struct {
|
||||
Output artifactCheckpointEnvelope `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
Output artifactCheckpointEnvelope `json:"output"`
|
||||
}
|
||||
|
||||
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
|
||||
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
||||
schema.JSONSchema = nil
|
||||
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
|
||||
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata), Diagnostics: cloneCheckpointDiagnostics(output.Diagnostics)}
|
||||
}
|
||||
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
|
||||
if len(outputs) == 0 {
|
||||
@@ -292,6 +291,20 @@ func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifa
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneCheckpointDiagnostics(values []pipeline.CheckpointDiagnostic) []pipeline.CheckpointDiagnostic {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]pipeline.CheckpointDiagnostic, len(values))
|
||||
for index, value := range values {
|
||||
cloned[index] = pipeline.CheckpointDiagnostic{
|
||||
Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{value.Diagnostic})[0],
|
||||
ValidatorKey: value.ValidatorKey,
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
|
||||
for i, v := range outputs {
|
||||
@@ -300,13 +313,12 @@ func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.Che
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any) binaryEnvelope {
|
||||
return binaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: contentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: cloneMetadata(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,18 +345,19 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
||||
cloned := make([]contracts.RejectedOutput, len(rejected))
|
||||
for index, item := range rejected {
|
||||
cloned[index] = item
|
||||
if item.Validation != nil {
|
||||
summary := artifacts.CloneValidationSummary(*item.Validation)
|
||||
cloned[index].Validation = &summary
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
@@ -452,13 +465,10 @@ func statusForRejected(rejected []contracts.RejectedOutput) StageStatus {
|
||||
return StatusSucceeded
|
||||
}
|
||||
|
||||
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
|
||||
func validationStatusString(rejected []contracts.RejectedOutput) string {
|
||||
if len(rejected) > 0 {
|
||||
return "rejected"
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
return "approved_with_warnings"
|
||||
}
|
||||
return "approved"
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestRootBasedRecorderOutputIsReusable(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("lane", "module", nil, nil, nil, nil); err != nil {
|
||||
if err := recorder.ExtractSucceeded("lane", "module", nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loader, err := NewFilesystemLoader(root, identity)
|
||||
@@ -44,7 +44,7 @@ func TestStepAwareRecorderAndLoaderIsolateLaneState(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.(pipeline.StepCheckpointRecorder).ExtractSucceededForStep("step-a", "lane", "module", nil, nil, nil, nil); err != nil {
|
||||
if err := recorder.(pipeline.StepCheckpointRecorder).ExtractSucceededForStep("step-a", "lane", "module", nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loader, err := NewFilesystemLoader(root, identity)
|
||||
@@ -87,7 +87,7 @@ func TestStepAwareCheckpointPreservesDistinctDotIdentities(t *testing.T) {
|
||||
LaneID: "lane", ModuleKey: "normalize-module", SourceID: "source", ChunkID: "chunk", ChunkRef: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
|
||||
Artifact: contracts.SerializedArtifact{Kind: "kind", Schema: contracts.ArtifactSchema{ID: "schema", Name: "Schema", Version: "1"}, MediaType: "application/json", Content: []byte(test.content)},
|
||||
}
|
||||
if err := stepRecorder.NormalizeSucceededForStep(test.stepID, "lane", "normalize-module", nil, artifact, nil); err != nil {
|
||||
if err := stepRecorder.NormalizeSucceededForStep(test.stepID, "lane", "normalize-module", nil, artifact); err != nil {
|
||||
t.Fatalf("record %q: %v", test.stepID, err)
|
||||
}
|
||||
}
|
||||
@@ -119,7 +119,7 @@ func TestStepAwareCheckpointPreservesDistinctDotIdentities(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckpointSchemaCompatibilityIdentifiers(t *testing.T) {
|
||||
if WorkspaceSchemaVersion != "notarius.workspace.v3" || WorkspaceSchemaVersionV2 != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
|
||||
if WorkspaceSchemaVersion != "notarius.workspace.v4" || WorkspaceSchemaVersionV3 != "notarius.workspace.v3" || WorkspaceSchemaVersionV2 != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
|
||||
t.Fatal("checkpoint schema identifiers are incorrect")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -340,6 +341,9 @@ func validateRecord(record pipeline.ChunkPlanRecord, requestedDigest string) err
|
||||
if record.SchemaVersion != SchemaVersion {
|
||||
return fmt.Errorf("schema_version %q is not supported", record.SchemaVersion)
|
||||
}
|
||||
if err := contracts.ValidateProducerDiagnostics(record.Diagnostics); err != nil {
|
||||
return fmt.Errorf("diagnostics: %w", err)
|
||||
}
|
||||
if _, err := digestPathSegment(requestedDigest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -281,7 +281,7 @@ func TestFilesystemStoreReportsInvalidRecordsAsRecoverable(t *testing.T) {
|
||||
return bytes.Replace(data, []byte(`{"schema_version"`), []byte(`{"SENTINEL_UNKNOWN_FIELD":true,"schema_version"`), 1)
|
||||
}},
|
||||
{name: "truncated JSON", mutate: func(data []byte) []byte { return data[:len(data)/2] }},
|
||||
{name: "legacy v1 record", mutate: replaceJSON(`notarius.chunk-plan.v2`, `notarius.chunk-plan.v1`)},
|
||||
{name: "legacy v2 record", mutate: replaceJSON(`notarius.chunk-plan.v3`, `notarius.chunk-plan.v2`)},
|
||||
{name: "source mismatch", mutate: replaceJSON(testSourceDigest, "sha256:"+strings.Repeat("b", 64))},
|
||||
{name: "plan digest mismatch", mutate: func(data []byte) []byte {
|
||||
prefix := []byte(`"plan_digest":"sha256:`)
|
||||
@@ -498,7 +498,13 @@ func testRecord(t *testing.T, value int) pipeline.ChunkPlanRecord {
|
||||
References: []artifacts.ReferenceProvenance{{Stage: "chunk", SlotName: "guide", OriginType: "file", OriginURI: "file:///guide.txt", Digest: "sha256:reference"}},
|
||||
Metadata: map[string]any{"prompt_id": "test/prompt", "enabled": true},
|
||||
},
|
||||
Warnings: []contracts.Warning{{Scope: "chunk/test", ReasonCode: "observed", Message: "warning"}},
|
||||
Diagnostics: []contracts.ProducerDiagnostic{{
|
||||
Disposition: contracts.DiagnosticDispositionWarning,
|
||||
Category: contracts.DiagnosticCategoryConfiguration,
|
||||
ReasonCode: "empty_reference",
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "reference", Message: "Reference was empty."}},
|
||||
}},
|
||||
CreatedAt: time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
84
internal/framework/contracts/completion_request_debug.go
Normal file
84
internal/framework/contracts/completion_request_debug.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// DebugStructuredCompletionRequest is the content-safe representation of a
|
||||
// structured completion request for ordinary diagnostics and summaries.
|
||||
// Detailed prompt material remains available only through the explicitly
|
||||
// requested LLM debug trace.
|
||||
type DebugStructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
InputCount int `json:"input_count"`
|
||||
VariableCount int `json:"variable_count"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Correction *DebugSemanticCorrection `json:"correction,omitempty"`
|
||||
}
|
||||
|
||||
// DebugSemanticCorrection records only safe correction metadata. It never
|
||||
// exposes the assistant response or user guidance text.
|
||||
type DebugSemanticCorrection struct {
|
||||
AssistantResponseBytes int `json:"assistant_response_bytes"`
|
||||
AssistantResponseDigest string `json:"assistant_response_digest"`
|
||||
UserGuidanceBytes int `json:"user_guidance_bytes"`
|
||||
UserGuidanceDigest string `json:"user_guidance_digest"`
|
||||
}
|
||||
|
||||
// DebugSummary returns a content-safe representation suitable for ordinary
|
||||
// diagnostics. It does not validate or retain correction content.
|
||||
func (request StructuredCompletionRequest) DebugSummary() DebugStructuredCompletionRequest {
|
||||
summary := DebugStructuredCompletionRequest{
|
||||
StageName: request.StageName,
|
||||
PromptID: request.PromptID,
|
||||
PromptVersion: request.PromptVersion,
|
||||
ProfileID: request.ProfileID,
|
||||
SessionID: request.SessionID,
|
||||
InputCount: len(request.Inputs),
|
||||
VariableCount: len(request.Vars),
|
||||
}
|
||||
if request.StructuredOutputRepairAttempts != nil {
|
||||
attempts := *request.StructuredOutputRepairAttempts
|
||||
summary.StructuredOutputRepairAttempts = &attempts
|
||||
}
|
||||
if request.Correction != nil {
|
||||
summary.Correction = request.Correction.DebugSummary()
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// DebugSummary returns content-safe correction metadata suitable for ordinary
|
||||
// diagnostics.
|
||||
func (correction *SemanticCorrection) DebugSummary() *DebugSemanticCorrection {
|
||||
if correction == nil {
|
||||
return nil
|
||||
}
|
||||
return &DebugSemanticCorrection{
|
||||
AssistantResponseBytes: len(correction.AssistantResponse),
|
||||
AssistantResponseDigest: debugContentDigest(correction.AssistantResponse),
|
||||
UserGuidanceBytes: len(correction.UserGuidance),
|
||||
UserGuidanceDigest: debugContentDigest([]byte(correction.UserGuidance)),
|
||||
}
|
||||
}
|
||||
|
||||
// String prevents ordinary request formatting from exposing correction
|
||||
// content. Use the explicitly requested debug trace for complete messages.
|
||||
func (request StructuredCompletionRequest) String() string {
|
||||
return fmt.Sprintf("%+v", request.DebugSummary())
|
||||
}
|
||||
|
||||
// GoString gives %#v formatting the same content-safe behavior as String.
|
||||
func (request StructuredCompletionRequest) GoString() string {
|
||||
return request.String()
|
||||
}
|
||||
|
||||
func debugContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -9,14 +9,15 @@ import (
|
||||
)
|
||||
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Correction *SemanticCorrection `json:"-"`
|
||||
}
|
||||
|
||||
type StructuredCompletionResponse struct {
|
||||
@@ -156,12 +157,14 @@ type ChunkRequest struct {
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Correction *SemanticCorrection `json:"-"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkPlanResult struct {
|
||||
Plan source.ChunkPlan `json:"plan"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Plan source.ChunkPlan `json:"plan"`
|
||||
Diagnostics []ProducerDiagnostic `json:"diagnostics,omitempty"`
|
||||
ModelCandidate *ModelCandidate `json:"-"`
|
||||
}
|
||||
|
||||
type Chunker interface {
|
||||
@@ -280,24 +283,19 @@ const (
|
||||
)
|
||||
|
||||
type ValidationResult struct {
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type Warning struct {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CorrectionGuidance string `json:"-"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Diagnostics []ProducerDiagnostic `json:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type OutputRequest struct {
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
|
||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Diagnostics DiagnosticCollection `json:"diagnostics,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
@@ -312,8 +310,7 @@ type OutputFile struct {
|
||||
}
|
||||
|
||||
type OutputResult struct {
|
||||
Files []OutputFile `json:"files,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Files []OutputFile `json:"files,omitempty"`
|
||||
}
|
||||
|
||||
type OutputEncoder interface {
|
||||
@@ -322,17 +319,18 @@ type OutputEncoder interface {
|
||||
}
|
||||
|
||||
type RejectedOutput struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Validation *artifacts.ValidationSummary `json:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type ManifestMetadataProvider interface {
|
||||
|
||||
178
internal/framework/contracts/correction.go
Normal file
178
internal/framework/contracts/correction.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
// CorrectionProtocol identifies how a producer can represent the model output
|
||||
// that directly controlled a candidate.
|
||||
type CorrectionProtocol string
|
||||
|
||||
const (
|
||||
CorrectionProtocolSingleResponseV1 CorrectionProtocol = "single_response_v1"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxValidationReasonCodeBytes = 128
|
||||
MaxValidationCorrectionGuidanceBytes = 4 * 1024
|
||||
MaxAssistantResponseBytes = 1 << 20
|
||||
MaxCorrectionGuidanceBytes = 64 * 1024
|
||||
MaxCorrectionContentBytes = MaxAssistantResponseBytes + MaxCorrectionGuidanceBytes
|
||||
)
|
||||
|
||||
func (protocol CorrectionProtocol) Validate() error {
|
||||
if protocol == CorrectionProtocolSingleResponseV1 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unsupported correction protocol %q", protocol)
|
||||
}
|
||||
|
||||
// SemanticCorrection carries the latest model response and application-owned
|
||||
// guidance for one fresh corrected request. Its content is sensitive and is
|
||||
// deliberately excluded from ordinary JSON serialization.
|
||||
type SemanticCorrection struct {
|
||||
AssistantResponse []byte `json:"-"`
|
||||
UserGuidance string `json:"-"`
|
||||
}
|
||||
|
||||
func NewSemanticCorrection(assistantResponse []byte, userGuidance string) (*SemanticCorrection, error) {
|
||||
correction := &SemanticCorrection{
|
||||
AssistantResponse: append([]byte(nil), assistantResponse...),
|
||||
UserGuidance: userGuidance,
|
||||
}
|
||||
if err := correction.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return correction, nil
|
||||
}
|
||||
|
||||
func CloneSemanticCorrection(correction *SemanticCorrection) (*SemanticCorrection, error) {
|
||||
if correction == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return NewSemanticCorrection(correction.AssistantResponse, correction.UserGuidance)
|
||||
}
|
||||
|
||||
// CloneStructuredCompletionRequest returns a request whose mutable values are
|
||||
// owned by the caller. It is suitable for clients that retain requests after
|
||||
// CompleteStructured returns.
|
||||
func CloneStructuredCompletionRequest(request StructuredCompletionRequest) (StructuredCompletionRequest, error) {
|
||||
correction, err := CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return StructuredCompletionRequest{}, fmt.Errorf("clone correction: %w", err)
|
||||
}
|
||||
vars, err := source.CloneMetadata(request.Vars)
|
||||
if err != nil {
|
||||
return StructuredCompletionRequest{}, fmt.Errorf("clone variables: %w", err)
|
||||
}
|
||||
request.Inputs = request.Inputs.Clone()
|
||||
request.Vars = vars
|
||||
request.Correction = correction
|
||||
if request.StructuredOutputRepairAttempts != nil {
|
||||
attempts := *request.StructuredOutputRepairAttempts
|
||||
request.StructuredOutputRepairAttempts = &attempts
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func (correction SemanticCorrection) Validate() error {
|
||||
if err := validateAssistantResponse(correction.AssistantResponse); err != nil {
|
||||
return fmt.Errorf("semantic correction assistant response: %w", err)
|
||||
}
|
||||
if err := validateBoundedText(correction.UserGuidance, MaxCorrectionGuidanceBytes, "semantic correction user guidance", false); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(correction.AssistantResponse)+len(correction.UserGuidance) > MaxCorrectionContentBytes {
|
||||
return errors.New("semantic correction content exceeds maximum length")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModelCandidate preserves the exact single model response that directly
|
||||
// controlled a producer result. It is attempt-local and never durable data.
|
||||
type ModelCandidate struct {
|
||||
Response []byte `json:"-"`
|
||||
Protocol CorrectionProtocol `json:"-"`
|
||||
}
|
||||
|
||||
func NewModelCandidate(response []byte, protocol CorrectionProtocol) (*ModelCandidate, error) {
|
||||
candidate := &ModelCandidate{Response: append([]byte(nil), response...), Protocol: protocol}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func CloneModelCandidate(candidate *ModelCandidate) (*ModelCandidate, error) {
|
||||
if candidate == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return NewModelCandidate(candidate.Response, candidate.Protocol)
|
||||
}
|
||||
|
||||
func (candidate ModelCandidate) Validate() error {
|
||||
if err := candidate.Protocol.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAssistantResponse(candidate.Response); err != nil {
|
||||
return fmt.Errorf("model candidate response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateValidationResult(result ValidationResult) error {
|
||||
if err := ValidateProducerDiagnostics(result.Diagnostics); err != nil {
|
||||
return fmt.Errorf("validation diagnostics: %w", err)
|
||||
}
|
||||
if !result.Approved && result.ReasonCode == "" {
|
||||
return errors.New("validation rejection reason code must not be empty")
|
||||
}
|
||||
if result.ReasonCode != "" {
|
||||
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !result.Approved && result.CorrectionGuidance == "" {
|
||||
return errors.New("validation rejection correction guidance must not be empty")
|
||||
}
|
||||
if result.CorrectionGuidance != "" {
|
||||
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAssistantResponse(response []byte) error {
|
||||
if len(response) > MaxAssistantResponseBytes {
|
||||
return errors.New("exceeds maximum length")
|
||||
}
|
||||
if !utf8.Valid(response) {
|
||||
return errors.New("must be valid UTF-8")
|
||||
}
|
||||
if len(strings.TrimSpace(string(response))) == 0 {
|
||||
return errors.New("must not be blank")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBoundedText(value string, maximum int, name string, optional bool) error {
|
||||
if value == "" && optional {
|
||||
return nil
|
||||
}
|
||||
if !utf8.ValidString(value) {
|
||||
return fmt.Errorf("%s must be valid UTF-8", name)
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("%s must not be blank", name)
|
||||
}
|
||||
if len(value) > maximum {
|
||||
return fmt.Errorf("%s exceeds maximum length", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
187
internal/framework/contracts/correction_test.go
Normal file
187
internal/framework/contracts/correction_test.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSemanticCorrectionOwnsValidatedContent(t *testing.T) {
|
||||
assistant := []byte(`{"items":["original"]}`)
|
||||
correction, err := NewSemanticCorrection(assistant, "Return one corrected replacement.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
assistant[0] = '['
|
||||
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||
t.Fatalf("assistant response = %q, want owned original content", got)
|
||||
}
|
||||
|
||||
clone, err := CloneSemanticCorrection(correction)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneSemanticCorrection() error = %v", err)
|
||||
}
|
||||
clone.AssistantResponse[0] = '['
|
||||
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||
t.Fatalf("source correction changed through clone = %q", got)
|
||||
}
|
||||
if nilClone, err := CloneSemanticCorrection(nil); err != nil || nilClone != nil {
|
||||
t.Fatalf("CloneSemanticCorrection(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(correction)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal correction: %v", err)
|
||||
}
|
||||
if string(encoded) != "{}" {
|
||||
t.Fatalf("correction JSON = %s, want no sensitive content", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
|
||||
tooLongAssistant := bytes.Repeat([]byte("a"), MaxAssistantResponseBytes+1)
|
||||
tooLongGuidance := strings.Repeat("a", MaxCorrectionGuidanceBytes+1)
|
||||
tooLongReason := strings.Repeat("a", MaxValidationReasonCodeBytes+1)
|
||||
tooLongValidationGuidance := strings.Repeat("a", MaxValidationCorrectionGuidanceBytes+1)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
call func() error
|
||||
}{
|
||||
{"blank assistant", func() error { _, err := NewSemanticCorrection([]byte(" \n"), "guidance"); return err }},
|
||||
{"invalid assistant utf8", func() error { _, err := NewSemanticCorrection([]byte{0xff}, "guidance"); return err }},
|
||||
{"oversized assistant", func() error { _, err := NewSemanticCorrection(tooLongAssistant, "guidance"); return err }},
|
||||
{"blank guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), " \t"); return err }},
|
||||
{"invalid guidance utf8", func() error { _, err := NewSemanticCorrection([]byte("response"), string([]byte{0xff})); return err }},
|
||||
{"oversized guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), tooLongGuidance); return err }},
|
||||
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
|
||||
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
|
||||
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
|
||||
{"missing rejection reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
|
||||
{"oversized reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"blank reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"invalid correction guidance utf8", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
|
||||
}},
|
||||
{"oversized correction guidance", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
|
||||
}},
|
||||
{"invalid diagnostics", func() error {
|
||||
return ValidateValidationResult(ValidationResult{Approved: true, Diagnostics: []ProducerDiagnostic{{}}})
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := test.call(); err == nil {
|
||||
t.Fatal("validation error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelCandidateOwnsValidatedResponse(t *testing.T) {
|
||||
response := []byte(`{"items":["original"]}`)
|
||||
candidate, err := NewModelCandidate(response, CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewModelCandidate() error = %v", err)
|
||||
}
|
||||
response[0] = '['
|
||||
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||
t.Fatalf("candidate response = %q, want owned original content", got)
|
||||
}
|
||||
clone, err := CloneModelCandidate(candidate)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneModelCandidate() error = %v", err)
|
||||
}
|
||||
clone.Response[0] = '['
|
||||
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||
t.Fatalf("source candidate changed through clone = %q", got)
|
||||
}
|
||||
if nilClone, err := CloneModelCandidate(nil); err != nil || nilClone != nil {
|
||||
t.Fatalf("CloneModelCandidate(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationResultAllowsAbsentOptionalCorrectionFields(t *testing.T) {
|
||||
if err := ValidateValidationResult(ValidationResult{Approved: true}); err != nil {
|
||||
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||
}
|
||||
if err := ValidateValidationResult(ValidationResult{ReasonCode: "invalid-evidence", CorrectionGuidance: "Provide source-backed evidence."}); err != nil {
|
||||
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||
}
|
||||
if err := CorrectionProtocol("").Validate(); err == nil {
|
||||
t.Fatal("empty correction protocol validation error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneStructuredCompletionRequestOwnsCorrection(t *testing.T) {
|
||||
correction, err := NewSemanticCorrection([]byte(`{"value":"original"}`), "Correct the value.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
attempts := 2
|
||||
request := StructuredCompletionRequest{
|
||||
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"labels": []string{"original"}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
Correction: correction,
|
||||
}
|
||||
clone, err := CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneStructuredCompletionRequest() error = %v", err)
|
||||
}
|
||||
correction.AssistantResponse[0] = '['
|
||||
request.Inputs["source"] = NewLLMInputMaterial("source", "application/json", []byte(`{"source":false}`), "", "")
|
||||
*request.StructuredOutputRepairAttempts = 7
|
||||
if got := string(clone.Correction.AssistantResponse); got != `{"value":"original"}` {
|
||||
t.Fatalf("cloned correction response = %q, want owned original content", got)
|
||||
}
|
||||
if got := string(clone.Inputs["source"].Content); got != `{"source":true}` {
|
||||
t.Fatalf("cloned input = %q, want owned original content", got)
|
||||
}
|
||||
if clone.StructuredOutputRepairAttempts == nil || *clone.StructuredOutputRepairAttempts != 2 {
|
||||
t.Fatalf("cloned repair attempts = %v, want 2", clone.StructuredOutputRepairAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredCompletionRequestDebugSummaryOmitsCorrectionContent(t *testing.T) {
|
||||
const assistantResponse = `{"secret":"assistant response"}`
|
||||
const userGuidance = "secret user guidance"
|
||||
correction, err := NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
request := StructuredCompletionRequest{
|
||||
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
Correction: correction,
|
||||
}
|
||||
|
||||
summary := request.DebugSummary()
|
||||
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||
t.Fatalf("debug summary = %#v, want input, variable, and correction metadata", summary)
|
||||
}
|
||||
if summary.Correction.AssistantResponseBytes != len(assistantResponse) || summary.Correction.UserGuidanceBytes != len(userGuidance) ||
|
||||
summary.Correction.AssistantResponseDigest == "" || summary.Correction.UserGuidanceDigest == "" {
|
||||
t.Fatalf("correction summary = %#v, want byte counts and digests", summary.Correction)
|
||||
}
|
||||
encoded, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal debug summary: %v", err)
|
||||
}
|
||||
for _, rendered := range []string{string(encoded), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request)} {
|
||||
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||
if strings.Contains(rendered, secret) {
|
||||
t.Fatalf("content-safe request rendering leaked %q: %s", secret, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
401
internal/framework/contracts/diagnostics.go
Normal file
401
internal/framework/contracts/diagnostics.go
Normal file
@@ -0,0 +1,401 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxDiagnosticReasonCodeBytes = 128
|
||||
MaxDiagnosticScopeBytes = 512
|
||||
MaxDiagnosticMessageBytes = 4 * 1024
|
||||
MaxDiagnosticSamples = 3
|
||||
MaxProducerDiagnosticGroups = 64
|
||||
)
|
||||
|
||||
// DiagnosticDisposition identifies the operator significance of a producer
|
||||
// finding. Warnings are reserved for process-level degradation or incomplete
|
||||
// configured work.
|
||||
type DiagnosticDisposition string
|
||||
|
||||
const (
|
||||
DiagnosticDispositionWarning DiagnosticDisposition = "warning"
|
||||
DiagnosticDispositionAdvisory DiagnosticDisposition = "advisory"
|
||||
DiagnosticDispositionObservation DiagnosticDisposition = "observation"
|
||||
)
|
||||
|
||||
// DiagnosticCategory gives a stable, bounded classification for a producer
|
||||
// finding.
|
||||
type DiagnosticCategory string
|
||||
|
||||
const (
|
||||
DiagnosticCategoryConfiguration DiagnosticCategory = "configuration"
|
||||
DiagnosticCategoryDegradation DiagnosticCategory = "degradation"
|
||||
DiagnosticCategoryValidationIncomplete DiagnosticCategory = "validation_incomplete"
|
||||
DiagnosticCategoryFallback DiagnosticCategory = "fallback"
|
||||
DiagnosticCategoryDataQuality DiagnosticCategory = "data_quality"
|
||||
DiagnosticCategoryNormalization DiagnosticCategory = "normalization"
|
||||
)
|
||||
|
||||
// DiagnosticOriginStage identifies the framework operation that promoted a
|
||||
// diagnostic. It is framework-owned rather than producer-owned.
|
||||
type DiagnosticOriginStage string
|
||||
|
||||
const (
|
||||
DiagnosticOriginStageReferences DiagnosticOriginStage = "references"
|
||||
DiagnosticOriginStageChunk DiagnosticOriginStage = "chunk"
|
||||
DiagnosticOriginStageExtract DiagnosticOriginStage = "extract"
|
||||
DiagnosticOriginStageMerge DiagnosticOriginStage = "merge"
|
||||
DiagnosticOriginStageNormalize DiagnosticOriginStage = "normalize"
|
||||
)
|
||||
|
||||
// DiagnosticSample is a bounded, safe example of a diagnostic occurrence.
|
||||
// Chunk identity is attached by the framework when it promotes a producer
|
||||
// diagnostic into a final group.
|
||||
type DiagnosticSample struct {
|
||||
Scope string `json:"scope"`
|
||||
Message string `json:"message"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex *int `json:"chunk_index,omitempty"`
|
||||
}
|
||||
|
||||
// ProducerDiagnostic is the locally grouped form returned by one producer or
|
||||
// validator. It intentionally has no pipeline origin.
|
||||
type ProducerDiagnostic struct {
|
||||
Disposition DiagnosticDisposition `json:"disposition"`
|
||||
Category DiagnosticCategory `json:"category"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
OccurrenceCount int `json:"occurrence_count"`
|
||||
Samples []DiagnosticSample `json:"samples"`
|
||||
OmittedSampleCount int `json:"omitted_sample_count"`
|
||||
}
|
||||
|
||||
// DiagnosticOrigin is framework-owned context used to distinguish findings
|
||||
// from different pipeline locations during final aggregation.
|
||||
type DiagnosticOrigin struct {
|
||||
Stage DiagnosticOriginStage `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ValidatorKey string `json:"validator_key,omitempty"`
|
||||
}
|
||||
|
||||
// DiagnosticGroup is a producer diagnostic after framework origin enrichment.
|
||||
type DiagnosticGroup struct {
|
||||
Disposition DiagnosticDisposition `json:"disposition"`
|
||||
Category DiagnosticCategory `json:"category"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Origin DiagnosticOrigin `json:"origin"`
|
||||
OccurrenceCount int `json:"occurrence_count"`
|
||||
Samples []DiagnosticSample `json:"samples"`
|
||||
OmittedSampleCount int `json:"omitted_sample_count"`
|
||||
}
|
||||
|
||||
// DiagnosticCollection is the grouped collection supplied to later durable
|
||||
// and presentation boundaries. Global aggregation policy is applied by the
|
||||
// framework before it reaches those boundaries.
|
||||
type DiagnosticCollection struct {
|
||||
Groups []DiagnosticGroup `json:"groups"`
|
||||
Truncated bool `json:"truncated"`
|
||||
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
|
||||
}
|
||||
|
||||
// DiagnosticProjection is the validated warning/non-warning view used by
|
||||
// durable and presentation boundaries. Occurrence totals are checked before
|
||||
// they leave the framework contract.
|
||||
type DiagnosticProjection struct {
|
||||
Warnings []DiagnosticGroup
|
||||
Diagnostics []DiagnosticGroup
|
||||
WarningOccurrenceCount int
|
||||
DiagnosticOccurrenceCount int
|
||||
}
|
||||
|
||||
// Validate checks a producer-local diagnostic against the public safety and
|
||||
// classification contract.
|
||||
func (diagnostic ProducerDiagnostic) Validate() error {
|
||||
return validateDiagnostic(
|
||||
diagnostic.Disposition,
|
||||
diagnostic.Category,
|
||||
diagnostic.ReasonCode,
|
||||
diagnostic.OccurrenceCount,
|
||||
diagnostic.Samples,
|
||||
diagnostic.OmittedSampleCount,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
// ValidateProducerDiagnostics validates the complete set returned by one
|
||||
// producer or validator result.
|
||||
func ValidateProducerDiagnostics(diagnostics []ProducerDiagnostic) error {
|
||||
if len(diagnostics) > MaxProducerDiagnosticGroups {
|
||||
return errors.New("producer diagnostics exceed maximum group count")
|
||||
}
|
||||
for index, diagnostic := range diagnostics {
|
||||
if err := diagnostic.Validate(); err != nil {
|
||||
return fmt.Errorf("producer diagnostic %d: %w", index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks framework-owned origin fields.
|
||||
func (origin DiagnosticOrigin) Validate() error {
|
||||
switch origin.Stage {
|
||||
case DiagnosticOriginStageReferences, DiagnosticOriginStageChunk, DiagnosticOriginStageExtract, DiagnosticOriginStageMerge, DiagnosticOriginStageNormalize:
|
||||
default:
|
||||
return errors.New("diagnostic origin stage is invalid")
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "diagnostic origin step ID", value: origin.StepID},
|
||||
{name: "diagnostic origin lane ID", value: origin.LaneID},
|
||||
{name: "diagnostic origin module key", value: origin.ModuleKey},
|
||||
{name: "diagnostic origin validator key", value: origin.ValidatorKey},
|
||||
} {
|
||||
if field.value != "" && (!utf8.ValidString(field.value) || strings.TrimSpace(field.value) == "") {
|
||||
return fmt.Errorf("%s must be valid nonblank UTF-8 when present", field.name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks a final origin-enriched group.
|
||||
func (group DiagnosticGroup) Validate() error {
|
||||
if err := group.Origin.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateDiagnostic(
|
||||
group.Disposition,
|
||||
group.Category,
|
||||
group.ReasonCode,
|
||||
group.OccurrenceCount,
|
||||
group.Samples,
|
||||
group.OmittedSampleCount,
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
// Validate checks the collection shape without imposing later global
|
||||
// aggregation limits.
|
||||
func (collection DiagnosticCollection) Validate() error {
|
||||
if collection.UnrepresentedOccurrenceCount < 0 {
|
||||
return errors.New("diagnostic collection unrepresented occurrence count must not be negative")
|
||||
}
|
||||
if !collection.Truncated && collection.UnrepresentedOccurrenceCount != 0 {
|
||||
return errors.New("diagnostic collection has unrepresented occurrences without truncation")
|
||||
}
|
||||
seen := make(map[diagnosticGroupKey]struct{}, len(collection.Groups))
|
||||
for index, group := range collection.Groups {
|
||||
if err := group.Validate(); err != nil {
|
||||
return fmt.Errorf("diagnostic group %d: %w", index, err)
|
||||
}
|
||||
key := diagnosticGroupKeyFromGroup(group)
|
||||
if _, exists := seen[key]; exists {
|
||||
return errors.New("diagnostic collection contains duplicate group identity")
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProjectDiagnosticCollection validates, partitions, and totals one finalized
|
||||
// collection. Unrepresented occurrences belong to the non-warning projection.
|
||||
func ProjectDiagnosticCollection(collection DiagnosticCollection) (DiagnosticProjection, error) {
|
||||
if err := collection.Validate(); err != nil {
|
||||
return DiagnosticProjection{}, err
|
||||
}
|
||||
projection := DiagnosticProjection{
|
||||
Warnings: make([]DiagnosticGroup, 0),
|
||||
Diagnostics: make([]DiagnosticGroup, 0),
|
||||
}
|
||||
for _, group := range collection.Groups {
|
||||
if group.Disposition == DiagnosticDispositionWarning {
|
||||
count, err := addDiagnosticOccurrences(projection.WarningOccurrenceCount, group.OccurrenceCount)
|
||||
if err != nil {
|
||||
return DiagnosticProjection{}, fmt.Errorf("warning occurrences: %w", err)
|
||||
}
|
||||
projection.WarningOccurrenceCount = count
|
||||
projection.Warnings = append(projection.Warnings, cloneDiagnosticGroup(group))
|
||||
continue
|
||||
}
|
||||
count, err := addDiagnosticOccurrences(projection.DiagnosticOccurrenceCount, group.OccurrenceCount)
|
||||
if err != nil {
|
||||
return DiagnosticProjection{}, fmt.Errorf("diagnostic occurrences: %w", err)
|
||||
}
|
||||
projection.DiagnosticOccurrenceCount = count
|
||||
projection.Diagnostics = append(projection.Diagnostics, cloneDiagnosticGroup(group))
|
||||
}
|
||||
count, err := addDiagnosticOccurrences(projection.DiagnosticOccurrenceCount, collection.UnrepresentedOccurrenceCount)
|
||||
if err != nil {
|
||||
return DiagnosticProjection{}, fmt.Errorf("diagnostic occurrences: %w", err)
|
||||
}
|
||||
projection.DiagnosticOccurrenceCount = count
|
||||
return projection, nil
|
||||
}
|
||||
|
||||
// CloneProducerDiagnostics returns independent diagnostic slice ownership.
|
||||
func CloneProducerDiagnostics(diagnostics []ProducerDiagnostic) []ProducerDiagnostic {
|
||||
if len(diagnostics) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]ProducerDiagnostic, len(diagnostics))
|
||||
for index, diagnostic := range diagnostics {
|
||||
cloned[index] = cloneProducerDiagnostic(diagnostic)
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
// CloneDiagnosticCollection returns independent collection ownership.
|
||||
func CloneDiagnosticCollection(collection DiagnosticCollection) DiagnosticCollection {
|
||||
groups := collection.Groups
|
||||
collection.Groups = make([]DiagnosticGroup, len(groups))
|
||||
for index, group := range groups {
|
||||
collection.Groups[index] = cloneDiagnosticGroup(group)
|
||||
}
|
||||
return collection
|
||||
}
|
||||
|
||||
func validateDiagnostic(disposition DiagnosticDisposition, category DiagnosticCategory, reasonCode string, occurrenceCount int, samples []DiagnosticSample, omittedSampleCount int, allowChunkContext bool) error {
|
||||
if !diagnosticCategoryAllowed(disposition, category) {
|
||||
return errors.New("diagnostic disposition and category combination is invalid")
|
||||
}
|
||||
if err := validateDiagnosticText(reasonCode, MaxDiagnosticReasonCodeBytes, "diagnostic reason code"); err != nil {
|
||||
return err
|
||||
}
|
||||
if occurrenceCount <= 0 {
|
||||
return errors.New("diagnostic occurrence count must be positive")
|
||||
}
|
||||
if len(samples) == 0 {
|
||||
return errors.New("diagnostic samples must not be empty")
|
||||
}
|
||||
if len(samples) > MaxDiagnosticSamples {
|
||||
return errors.New("diagnostic samples exceed maximum count")
|
||||
}
|
||||
seen := make(map[diagnosticSampleKey]struct{}, len(samples))
|
||||
for index, sample := range samples {
|
||||
if err := validateDiagnosticSample(sample, allowChunkContext); err != nil {
|
||||
return fmt.Errorf("diagnostic sample %d: %w", index, err)
|
||||
}
|
||||
key := diagnosticSampleKeyFromSample(sample)
|
||||
if _, exists := seen[key]; exists {
|
||||
return errors.New("diagnostic samples must be distinct")
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
if occurrenceCount < len(samples) {
|
||||
return errors.New("diagnostic occurrence count is smaller than sample count")
|
||||
}
|
||||
if omittedSampleCount != occurrenceCount-len(samples) {
|
||||
return errors.New("diagnostic omitted sample count is inconsistent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func diagnosticCategoryAllowed(disposition DiagnosticDisposition, category DiagnosticCategory) bool {
|
||||
switch disposition {
|
||||
case DiagnosticDispositionWarning:
|
||||
return category == DiagnosticCategoryConfiguration || category == DiagnosticCategoryDegradation || category == DiagnosticCategoryValidationIncomplete || category == DiagnosticCategoryFallback
|
||||
case DiagnosticDispositionAdvisory:
|
||||
return category == DiagnosticCategoryDataQuality
|
||||
case DiagnosticDispositionObservation:
|
||||
return category == DiagnosticCategoryNormalization
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validateDiagnosticSample(sample DiagnosticSample, allowChunkContext bool) error {
|
||||
if err := validateDiagnosticText(sample.Scope, MaxDiagnosticScopeBytes, "diagnostic sample scope"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDiagnosticText(sample.Message, MaxDiagnosticMessageBytes, "diagnostic sample message"); err != nil {
|
||||
return err
|
||||
}
|
||||
if !allowChunkContext && (sample.ChunkID != "" || sample.ChunkIndex != nil) {
|
||||
return errors.New("producer diagnostic sample must not include framework chunk context")
|
||||
}
|
||||
if sample.ChunkID != "" && (!utf8.ValidString(sample.ChunkID) || strings.TrimSpace(sample.ChunkID) == "") {
|
||||
return errors.New("diagnostic sample chunk ID must be valid nonblank UTF-8 when present")
|
||||
}
|
||||
if sample.ChunkIndex != nil && *sample.ChunkIndex < 0 {
|
||||
return errors.New("diagnostic sample chunk index must not be negative")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDiagnosticText(value string, maximum int, name string) error {
|
||||
if !utf8.ValidString(value) {
|
||||
return fmt.Errorf("%s must be valid UTF-8", name)
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("%s must not be blank", name)
|
||||
}
|
||||
if len(value) > maximum {
|
||||
return fmt.Errorf("%s exceeds maximum length", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneProducerDiagnostic(diagnostic ProducerDiagnostic) ProducerDiagnostic {
|
||||
diagnostic.Samples = cloneDiagnosticSamples(diagnostic.Samples)
|
||||
return diagnostic
|
||||
}
|
||||
|
||||
func cloneDiagnosticGroup(group DiagnosticGroup) DiagnosticGroup {
|
||||
group.Samples = cloneDiagnosticSamples(group.Samples)
|
||||
return group
|
||||
}
|
||||
|
||||
func addDiagnosticOccurrences(current, incoming int) (int, error) {
|
||||
if incoming > int(^uint(0)>>1)-current {
|
||||
return 0, errors.New("occurrence count overflow")
|
||||
}
|
||||
return current + incoming, nil
|
||||
}
|
||||
|
||||
func cloneDiagnosticSamples(samples []DiagnosticSample) []DiagnosticSample {
|
||||
if len(samples) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]DiagnosticSample, len(samples))
|
||||
for index, sample := range samples {
|
||||
if sample.ChunkIndex != nil {
|
||||
chunkIndex := *sample.ChunkIndex
|
||||
sample.ChunkIndex = &chunkIndex
|
||||
}
|
||||
cloned[index] = sample
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
type diagnosticSampleKey struct {
|
||||
scope string
|
||||
message string
|
||||
chunkID string
|
||||
chunkIndex int
|
||||
hasChunkIndex bool
|
||||
}
|
||||
|
||||
func diagnosticSampleKeyFromSample(sample DiagnosticSample) diagnosticSampleKey {
|
||||
key := diagnosticSampleKey{scope: sample.Scope, message: sample.Message, chunkID: sample.ChunkID}
|
||||
if sample.ChunkIndex != nil {
|
||||
key.chunkIndex = *sample.ChunkIndex
|
||||
key.hasChunkIndex = true
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
type diagnosticGroupKey struct {
|
||||
disposition DiagnosticDisposition
|
||||
category DiagnosticCategory
|
||||
reasonCode string
|
||||
origin DiagnosticOrigin
|
||||
}
|
||||
|
||||
func diagnosticGroupKeyFromGroup(group DiagnosticGroup) diagnosticGroupKey {
|
||||
return diagnosticGroupKey{disposition: group.Disposition, category: group.Category, reasonCode: group.ReasonCode, origin: group.Origin}
|
||||
}
|
||||
198
internal/framework/contracts/diagnostics_test.go
Normal file
198
internal/framework/contracts/diagnostics_test.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProducerDiagnosticValidationAcceptsClassificationMatrix(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
disposition DiagnosticDisposition
|
||||
category DiagnosticCategory
|
||||
}{
|
||||
{name: "configuration warning", disposition: DiagnosticDispositionWarning, category: DiagnosticCategoryConfiguration},
|
||||
{name: "degradation warning", disposition: DiagnosticDispositionWarning, category: DiagnosticCategoryDegradation},
|
||||
{name: "incomplete validation warning", disposition: DiagnosticDispositionWarning, category: DiagnosticCategoryValidationIncomplete},
|
||||
{name: "fallback warning", disposition: DiagnosticDispositionWarning, category: DiagnosticCategoryFallback},
|
||||
{name: "quality advisory", disposition: DiagnosticDispositionAdvisory, category: DiagnosticCategoryDataQuality},
|
||||
{name: "normalization observation", disposition: DiagnosticDispositionObservation, category: DiagnosticCategoryNormalization},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
diagnostic := validProducerDiagnostic()
|
||||
diagnostic.Disposition = test.disposition
|
||||
diagnostic.Category = test.category
|
||||
if err := diagnostic.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerDiagnosticValidationRejectsInvalidFieldsAndCounts(t *testing.T) {
|
||||
tooLongReason := strings.Repeat("r", MaxDiagnosticReasonCodeBytes+1)
|
||||
tooLongScope := strings.Repeat("s", MaxDiagnosticScopeBytes+1)
|
||||
tooLongMessage := strings.Repeat("m", MaxDiagnosticMessageBytes+1)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*ProducerDiagnostic)
|
||||
}{
|
||||
{name: "invalid classification", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Category = DiagnosticCategoryDataQuality }},
|
||||
{name: "blank reason", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.ReasonCode = " \t" }},
|
||||
{name: "invalid reason UTF-8", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.ReasonCode = string([]byte{0xff}) }},
|
||||
{name: "oversized reason", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.ReasonCode = tooLongReason }},
|
||||
{name: "blank scope", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Scope = "\n" }},
|
||||
{name: "invalid scope UTF-8", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Scope = string([]byte{0xff}) }},
|
||||
{name: "oversized scope", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Scope = tooLongScope }},
|
||||
{name: "blank message", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Message = " " }},
|
||||
{name: "invalid message UTF-8", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Message = string([]byte{0xff}) }},
|
||||
{name: "oversized message", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.Samples[0].Message = tooLongMessage }},
|
||||
{name: "zero occurrences", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.OccurrenceCount = 0 }},
|
||||
{name: "missing samples", mutate: func(diagnostic *ProducerDiagnostic) {
|
||||
diagnostic.Samples = nil
|
||||
diagnostic.OmittedSampleCount = diagnostic.OccurrenceCount
|
||||
}},
|
||||
{name: "too many samples", mutate: func(diagnostic *ProducerDiagnostic) {
|
||||
diagnostic.OccurrenceCount = 4
|
||||
diagnostic.Samples = []DiagnosticSample{{Scope: "one", Message: "one"}, {Scope: "two", Message: "two"}, {Scope: "three", Message: "three"}, {Scope: "four", Message: "four"}}
|
||||
diagnostic.OmittedSampleCount = 0
|
||||
}},
|
||||
{name: "duplicate samples", mutate: func(diagnostic *ProducerDiagnostic) {
|
||||
diagnostic.OccurrenceCount = 2
|
||||
diagnostic.Samples = []DiagnosticSample{{Scope: "scope", Message: "message"}, {Scope: "scope", Message: "message"}}
|
||||
diagnostic.OmittedSampleCount = 0
|
||||
}},
|
||||
{name: "inconsistent omission", mutate: func(diagnostic *ProducerDiagnostic) { diagnostic.OmittedSampleCount = 1 }},
|
||||
{name: "producer chunk context", mutate: func(diagnostic *ProducerDiagnostic) { chunkIndex := 0; diagnostic.Samples[0].ChunkIndex = &chunkIndex }},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
diagnostic := validProducerDiagnostic()
|
||||
test.mutate(&diagnostic)
|
||||
if err := diagnostic.Validate(); err == nil {
|
||||
t.Fatal("Validate() error = nil, want invalid diagnostic error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateProducerDiagnosticsEnforcesLocalGroupBound(t *testing.T) {
|
||||
diagnostics := make([]ProducerDiagnostic, MaxProducerDiagnosticGroups)
|
||||
for index := range diagnostics {
|
||||
diagnostics[index] = validProducerDiagnostic()
|
||||
diagnostics[index].ReasonCode = "reason-" + string(rune('a'+index))
|
||||
}
|
||||
if err := ValidateProducerDiagnostics(diagnostics); err != nil {
|
||||
t.Fatalf("ValidateProducerDiagnostics() error = %v", err)
|
||||
}
|
||||
diagnostics = append(diagnostics, validProducerDiagnostic())
|
||||
if err := ValidateProducerDiagnostics(diagnostics); err == nil {
|
||||
t.Fatal("ValidateProducerDiagnostics() error = nil, want excessive-group error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticGroupValidationPreservesChunkIndexZero(t *testing.T) {
|
||||
chunkIndex := 0
|
||||
group := DiagnosticGroup{
|
||||
Disposition: DiagnosticDispositionAdvisory,
|
||||
Category: DiagnosticCategoryDataQuality,
|
||||
ReasonCode: "unresolved",
|
||||
Origin: DiagnosticOrigin{Stage: DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells", ValidatorKey: "dnd/spells/source-relatedness"},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []DiagnosticSample{{Scope: "spells[0]", Message: "Spell was not found", ChunkID: "chunk-1", ChunkIndex: &chunkIndex}},
|
||||
}
|
||||
if err := group.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
collection := DiagnosticCollection{Groups: []DiagnosticGroup{group}}
|
||||
if err := collection.Validate(); err != nil {
|
||||
t.Fatalf("DiagnosticCollection.Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticGroupRejectsRepeatedSampleWithEqualChunkIndex(t *testing.T) {
|
||||
firstIndex := 0
|
||||
secondIndex := 0
|
||||
group := DiagnosticGroup{
|
||||
Disposition: DiagnosticDispositionAdvisory,
|
||||
Category: DiagnosticCategoryDataQuality,
|
||||
ReasonCode: "unresolved",
|
||||
Origin: DiagnosticOrigin{Stage: DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells"},
|
||||
OccurrenceCount: 2,
|
||||
Samples: []DiagnosticSample{
|
||||
{Scope: "spells[0]", Message: "Spell was not found", ChunkID: "chunk-1", ChunkIndex: &firstIndex},
|
||||
{Scope: "spells[0]", Message: "Spell was not found", ChunkID: "chunk-1", ChunkIndex: &secondIndex},
|
||||
},
|
||||
}
|
||||
if err := group.Validate(); err == nil {
|
||||
t.Fatal("Validate() error = nil, want duplicate sample error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneDiagnosticCollectionOwnsGroupsAndChunkIndex(t *testing.T) {
|
||||
chunkIndex := 0
|
||||
collection := DiagnosticCollection{Groups: []DiagnosticGroup{{
|
||||
Disposition: DiagnosticDispositionAdvisory,
|
||||
Category: DiagnosticCategoryDataQuality,
|
||||
ReasonCode: "unresolved",
|
||||
Origin: DiagnosticOrigin{Stage: DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells"},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []DiagnosticSample{{Scope: "spells[0]", Message: "Spell was not found", ChunkIndex: &chunkIndex}},
|
||||
}}}
|
||||
cloned := CloneDiagnosticCollection(collection)
|
||||
collection.Groups[0].Samples[0].Message = "changed"
|
||||
*collection.Groups[0].Samples[0].ChunkIndex = 1
|
||||
if got := cloned.Groups[0].Samples[0]; got.Message != "Spell was not found" || got.ChunkIndex == nil || *got.ChunkIndex != 0 {
|
||||
t.Fatalf("cloned sample = %#v, want independently owned original", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectDiagnosticCollectionPartitionsAndChecksTotals(t *testing.T) {
|
||||
warning := validDiagnosticGroup(DiagnosticDispositionWarning, DiagnosticCategoryFallback, "fallback", 2)
|
||||
diagnostic := validDiagnosticGroup(DiagnosticDispositionAdvisory, DiagnosticCategoryDataQuality, "quality", 3)
|
||||
collection := DiagnosticCollection{
|
||||
Groups: []DiagnosticGroup{warning, diagnostic},
|
||||
Truncated: true,
|
||||
UnrepresentedOccurrenceCount: 4,
|
||||
}
|
||||
projection, err := ProjectDiagnosticCollection(collection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(projection.Warnings) != 1 || len(projection.Diagnostics) != 1 || projection.WarningOccurrenceCount != 2 || projection.DiagnosticOccurrenceCount != 7 {
|
||||
t.Fatalf("projection = %#v, want partitioned exact totals", projection)
|
||||
}
|
||||
collection.Groups[0].Samples[0].Message = "mutated"
|
||||
if projection.Warnings[0].Samples[0].Message != "message" {
|
||||
t.Fatal("projection retained caller-owned sample storage")
|
||||
}
|
||||
|
||||
overflow := DiagnosticCollection{Groups: []DiagnosticGroup{
|
||||
validDiagnosticGroup(DiagnosticDispositionWarning, DiagnosticCategoryFallback, "first", int(^uint(0)>>1)),
|
||||
validDiagnosticGroup(DiagnosticDispositionWarning, DiagnosticCategoryFallback, "second", 1),
|
||||
}}
|
||||
if _, err := ProjectDiagnosticCollection(overflow); err == nil {
|
||||
t.Fatal("ProjectDiagnosticCollection() overflow error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func validDiagnosticGroup(disposition DiagnosticDisposition, category DiagnosticCategory, reason string, occurrences int) DiagnosticGroup {
|
||||
return DiagnosticGroup{
|
||||
Disposition: disposition,
|
||||
Category: category,
|
||||
ReasonCode: reason,
|
||||
Origin: DiagnosticOrigin{Stage: DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
||||
OccurrenceCount: occurrences,
|
||||
Samples: []DiagnosticSample{{Scope: "scope", Message: "message"}},
|
||||
OmittedSampleCount: occurrences - 1,
|
||||
}
|
||||
}
|
||||
|
||||
func validProducerDiagnostic() ProducerDiagnostic {
|
||||
return ProducerDiagnostic{
|
||||
Disposition: DiagnosticDispositionWarning,
|
||||
Category: DiagnosticCategoryConfiguration,
|
||||
ReasonCode: "empty_reference",
|
||||
OccurrenceCount: 1,
|
||||
Samples: []DiagnosticSample{{Scope: "references.glossary", Message: "Reference is empty"}},
|
||||
}
|
||||
}
|
||||
@@ -42,12 +42,14 @@ type TypedExtractionRequest struct {
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Correction *SemanticCorrection
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedExtractionResult[T any] struct {
|
||||
Value T
|
||||
Warnings []Warning
|
||||
Value T
|
||||
Diagnostics []ProducerDiagnostic
|
||||
ModelCandidate *ModelCandidate
|
||||
}
|
||||
|
||||
type Extractor[T any] interface {
|
||||
@@ -65,12 +67,14 @@ type TypedMergeRequest[T any] struct {
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Correction *SemanticCorrection
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedMergeResult[T any] struct {
|
||||
Value T
|
||||
Warnings []Warning
|
||||
Value T
|
||||
Diagnostics []ProducerDiagnostic
|
||||
ModelCandidate *ModelCandidate
|
||||
}
|
||||
|
||||
type Merger[T any] interface {
|
||||
@@ -87,28 +91,32 @@ type TypedNormalizeRequest[T any] struct {
|
||||
References ReferenceSet
|
||||
LLMProfile string
|
||||
StructuredOutputRepairAttempts *int
|
||||
Correction *SemanticCorrection
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type TypedNormalizeResult[T any] struct {
|
||||
Value T
|
||||
Warnings []Warning
|
||||
Retry *NormalizeRetry
|
||||
Value T
|
||||
Diagnostics []ProducerDiagnostic
|
||||
Retry *NormalizeRetry
|
||||
ModelCandidate *ModelCandidate
|
||||
}
|
||||
|
||||
// Normalize retry diagnostic limits bound module-provided values before the
|
||||
// framework persists them in debug artifacts.
|
||||
// Normalize retry limits bound module-provided control and diagnostic text
|
||||
// before the framework consumes or records it.
|
||||
const (
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryCorrectionGuidanceBytes = 4096
|
||||
)
|
||||
|
||||
// NormalizeRetry asks the framework to retry normalization while retaining a
|
||||
// safe candidate for acceptance if the retry budget is exhausted.
|
||||
type NormalizeRetry struct {
|
||||
ReasonCode string
|
||||
Message string
|
||||
FallbackWarnings []Warning
|
||||
ReasonCode string
|
||||
Message string
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []ProducerDiagnostic
|
||||
}
|
||||
|
||||
type Normalizer[T any] interface {
|
||||
|
||||
164
internal/framework/diagnostics/aggregator.go
Normal file
164
internal/framework/diagnostics/aggregator.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxWarningGroups = 128
|
||||
MaxNonWarningGroups = 256
|
||||
)
|
||||
|
||||
// Aggregator merges origin-enriched diagnostics in caller-supplied canonical
|
||||
// order. Its zero value is ready for use.
|
||||
type Aggregator struct {
|
||||
groups []contracts.DiagnosticGroup
|
||||
indices map[groupKey]int
|
||||
warningGroups int
|
||||
nonWarningGroups int
|
||||
warningOccurrences int
|
||||
nonWarningOccurrences int
|
||||
unrepresentedOccurrences int
|
||||
}
|
||||
|
||||
// Add validates and incorporates one final diagnostic group. Actionable
|
||||
// warnings cannot overflow; later non-warning groups are represented by exact
|
||||
// unrepresented-occurrence metadata once their fixed bound is reached.
|
||||
func (aggregator *Aggregator) Add(group contracts.DiagnosticGroup) error {
|
||||
if err := group.Validate(); err != nil {
|
||||
return fmt.Errorf("diagnostic group: %w", err)
|
||||
}
|
||||
if aggregator.indices == nil {
|
||||
aggregator.indices = make(map[groupKey]int)
|
||||
}
|
||||
key := groupKeyFromGroup(group)
|
||||
if index, exists := aggregator.indices[key]; exists {
|
||||
if err := aggregator.checkOccurrenceTotal(group.Disposition, group.OccurrenceCount); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := aggregator.merge(index, group); err != nil {
|
||||
return err
|
||||
}
|
||||
aggregator.addOccurrenceTotal(group.Disposition, group.OccurrenceCount)
|
||||
return nil
|
||||
}
|
||||
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||
if aggregator.warningGroups >= MaxWarningGroups {
|
||||
return errors.New("diagnostic warning groups exceed maximum count")
|
||||
}
|
||||
} else if aggregator.nonWarningGroups >= MaxNonWarningGroups {
|
||||
if err := aggregator.checkOccurrenceTotal(group.Disposition, group.OccurrenceCount); err != nil {
|
||||
return err
|
||||
}
|
||||
aggregator.addOccurrenceTotal(group.Disposition, group.OccurrenceCount)
|
||||
return aggregator.addUnrepresented(group.OccurrenceCount)
|
||||
}
|
||||
if err := aggregator.checkOccurrenceTotal(group.Disposition, group.OccurrenceCount); err != nil {
|
||||
return err
|
||||
}
|
||||
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||
aggregator.warningGroups++
|
||||
} else {
|
||||
aggregator.nonWarningGroups++
|
||||
}
|
||||
aggregator.addOccurrenceTotal(group.Disposition, group.OccurrenceCount)
|
||||
aggregator.indices[key] = len(aggregator.groups)
|
||||
aggregator.groups = append(aggregator.groups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{group}}).Groups[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) checkOccurrenceTotal(disposition contracts.DiagnosticDisposition, count int) error {
|
||||
current := aggregator.nonWarningOccurrences
|
||||
if disposition == contracts.DiagnosticDispositionWarning {
|
||||
current = aggregator.warningOccurrences
|
||||
}
|
||||
if count > maximumInt()-current {
|
||||
return errors.New("diagnostic occurrence count overflow")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) addOccurrenceTotal(disposition contracts.DiagnosticDisposition, count int) {
|
||||
if disposition == contracts.DiagnosticDispositionWarning {
|
||||
aggregator.warningOccurrences += count
|
||||
return
|
||||
}
|
||||
aggregator.nonWarningOccurrences += count
|
||||
}
|
||||
|
||||
// Collection returns an independently owned grouped result in first-occurrence
|
||||
// order.
|
||||
func (aggregator *Aggregator) Collection() contracts.DiagnosticCollection {
|
||||
if aggregator == nil {
|
||||
return contracts.DiagnosticCollection{}
|
||||
}
|
||||
return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{
|
||||
Groups: aggregator.groups,
|
||||
Truncated: aggregator.unrepresentedOccurrences > 0,
|
||||
UnrepresentedOccurrenceCount: aggregator.unrepresentedOccurrences,
|
||||
})
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) merge(index int, incoming contracts.DiagnosticGroup) error {
|
||||
current := &aggregator.groups[index]
|
||||
if incoming.OccurrenceCount > maximumInt()-current.OccurrenceCount {
|
||||
return errors.New("diagnostic occurrence count overflow")
|
||||
}
|
||||
current.OccurrenceCount += incoming.OccurrenceCount
|
||||
for _, sample := range incoming.Samples {
|
||||
if len(current.Samples) == contracts.MaxDiagnosticSamples || containsGroupSample(current.Samples, sample) {
|
||||
continue
|
||||
}
|
||||
current.Samples = append(current.Samples, cloneGroupSample(sample))
|
||||
}
|
||||
current.OmittedSampleCount = current.OccurrenceCount - len(current.Samples)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) addUnrepresented(count int) error {
|
||||
if count > maximumInt()-aggregator.unrepresentedOccurrences {
|
||||
return errors.New("diagnostic unrepresented occurrence count overflow")
|
||||
}
|
||||
aggregator.unrepresentedOccurrences += count
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsGroupSample(samples []contracts.DiagnosticSample, candidate contracts.DiagnosticSample) bool {
|
||||
for _, sample := range samples {
|
||||
if sample.Scope != candidate.Scope || sample.Message != candidate.Message || sample.ChunkID != candidate.ChunkID {
|
||||
continue
|
||||
}
|
||||
if sample.ChunkIndex == nil || candidate.ChunkIndex == nil {
|
||||
if sample.ChunkIndex == candidate.ChunkIndex {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if *sample.ChunkIndex == *candidate.ChunkIndex {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cloneGroupSample(sample contracts.DiagnosticSample) contracts.DiagnosticSample {
|
||||
if sample.ChunkIndex != nil {
|
||||
value := *sample.ChunkIndex
|
||||
sample.ChunkIndex = &value
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
type groupKey struct {
|
||||
disposition contracts.DiagnosticDisposition
|
||||
category contracts.DiagnosticCategory
|
||||
reasonCode string
|
||||
origin contracts.DiagnosticOrigin
|
||||
}
|
||||
|
||||
func groupKeyFromGroup(group contracts.DiagnosticGroup) groupKey {
|
||||
return groupKey{disposition: group.Disposition, category: group.Category, reasonCode: group.ReasonCode, origin: group.Origin}
|
||||
}
|
||||
89
internal/framework/diagnostics/aggregator_test.go
Normal file
89
internal/framework/diagnostics/aggregator_test.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestAggregatorMergesByOriginAndPreservesFirstOccurrenceOrder(t *testing.T) {
|
||||
aggregator := Aggregator{}
|
||||
for _, group := range []contracts.DiagnosticGroup{
|
||||
groupForAggregation("first", "message one", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "dnd/spells"),
|
||||
groupForAggregation("second", "message two", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "dnd/items"),
|
||||
groupForAggregation("third", "message three", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "dnd/spells"),
|
||||
} {
|
||||
if err := aggregator.Add(group); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
}
|
||||
collection := aggregator.Collection()
|
||||
if len(collection.Groups) != 2 {
|
||||
t.Fatalf("group count = %d, want 2", len(collection.Groups))
|
||||
}
|
||||
if got := []string{collection.Groups[0].Origin.ModuleKey, collection.Groups[1].Origin.ModuleKey}; !equalStrings(got, []string{"dnd/spells", "dnd/items"}) {
|
||||
t.Fatalf("group order = %#v, want first occurrence order", got)
|
||||
}
|
||||
if collection.Groups[0].OccurrenceCount != 2 || collection.Groups[0].OmittedSampleCount != 0 {
|
||||
t.Fatalf("merged group = %#v, want two represented occurrences", collection.Groups[0])
|
||||
}
|
||||
if got := []string{collection.Groups[0].Samples[0].Scope, collection.Groups[0].Samples[1].Scope}; !equalStrings(got, []string{"first", "third"}) {
|
||||
t.Fatalf("merged samples = %#v, want first occurrence order", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregatorEnforcesWarningBoundAndTruncatesOnlyNonWarnings(t *testing.T) {
|
||||
warnings := Aggregator{}
|
||||
for index := 0; index < MaxWarningGroups; index++ {
|
||||
group := groupForAggregation("scope", "message", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "module")
|
||||
group.ReasonCode = "warning-" + string(rune('a'+index))
|
||||
if err := warnings.Add(group); err != nil {
|
||||
t.Fatalf("warning Add(%d) error = %v", index, err)
|
||||
}
|
||||
}
|
||||
if err := warnings.Add(groupForAggregation("overflow", "overflow", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "overflow")); err == nil {
|
||||
t.Fatal("warning overflow error = nil, want error")
|
||||
}
|
||||
|
||||
nonWarnings := Aggregator{}
|
||||
for index := 0; index < MaxNonWarningGroups+3; index++ {
|
||||
group := groupForAggregation("scope", "message", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "module")
|
||||
group.ReasonCode = "advisory-" + string(rune('a'+index))
|
||||
if err := nonWarnings.Add(group); err != nil {
|
||||
t.Fatalf("non-warning Add(%d) error = %v", index, err)
|
||||
}
|
||||
}
|
||||
collection := nonWarnings.Collection()
|
||||
if len(collection.Groups) != MaxNonWarningGroups || !collection.Truncated || collection.UnrepresentedOccurrenceCount != 3 {
|
||||
t.Fatalf("collection = %#v, want bounded non-warning groups and three unrepresented occurrences", collection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregatorRejectsOccurrenceTotalOverflow(t *testing.T) {
|
||||
aggregator := Aggregator{}
|
||||
first := groupForAggregation("first", "first", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "first")
|
||||
first.OccurrenceCount = maximumInt()
|
||||
first.OmittedSampleCount = maximumInt() - 1
|
||||
if err := aggregator.Add(first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := groupForAggregation("second", "second", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "second")
|
||||
if err := aggregator.Add(second); err == nil {
|
||||
t.Fatal("Add() occurrence total overflow error = nil")
|
||||
}
|
||||
collection := aggregator.Collection()
|
||||
if len(collection.Groups) != 1 || collection.Groups[0].OccurrenceCount != maximumInt() {
|
||||
t.Fatalf("collection changed after rejected overflow = %#v", collection)
|
||||
}
|
||||
}
|
||||
|
||||
func groupForAggregation(scope string, message string, disposition contracts.DiagnosticDisposition, category contracts.DiagnosticCategory, module string) contracts.DiagnosticGroup {
|
||||
return contracts.DiagnosticGroup{
|
||||
Disposition: disposition,
|
||||
Category: category,
|
||||
ReasonCode: "source_unrelated",
|
||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: module},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}},
|
||||
}
|
||||
}
|
||||
93
internal/framework/diagnostics/collector.go
Normal file
93
internal/framework/diagnostics/collector.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Package diagnostics provides bounded local grouping for producer and
|
||||
// validator diagnostic results.
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// Collector merges local producer diagnostics by their semantic identity. Its
|
||||
// zero value is ready for use.
|
||||
type Collector struct {
|
||||
diagnostics []contracts.ProducerDiagnostic
|
||||
indices map[key]int
|
||||
}
|
||||
|
||||
// NewCollector returns an empty local diagnostic collector.
|
||||
func NewCollector() *Collector {
|
||||
return &Collector{}
|
||||
}
|
||||
|
||||
// Add validates and merges one producer diagnostic. Every occurrence remains
|
||||
// counted, while the first three distinct samples in input order are retained.
|
||||
func (collector *Collector) Add(diagnostic contracts.ProducerDiagnostic) error {
|
||||
if err := diagnostic.Validate(); err != nil {
|
||||
return fmt.Errorf("producer diagnostic: %w", err)
|
||||
}
|
||||
if collector.indices == nil {
|
||||
collector.indices = make(map[key]int)
|
||||
}
|
||||
diagnosticKey := key{disposition: diagnostic.Disposition, category: diagnostic.Category, reasonCode: diagnostic.ReasonCode}
|
||||
index, exists := collector.indices[diagnosticKey]
|
||||
if !exists {
|
||||
if len(collector.diagnostics) >= contracts.MaxProducerDiagnosticGroups {
|
||||
return errors.New("producer diagnostics exceed maximum group count")
|
||||
}
|
||||
collector.indices[diagnosticKey] = len(collector.diagnostics)
|
||||
collector.diagnostics = append(collector.diagnostics, contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
current := &collector.diagnostics[index]
|
||||
if diagnostic.OccurrenceCount > maximumInt()-current.OccurrenceCount {
|
||||
return errors.New("producer diagnostic occurrence count overflow")
|
||||
}
|
||||
current.OccurrenceCount += diagnostic.OccurrenceCount
|
||||
for _, sample := range diagnostic.Samples {
|
||||
if len(current.Samples) == contracts.MaxDiagnosticSamples || containsSample(current.Samples, sample) {
|
||||
continue
|
||||
}
|
||||
current.Samples = append(current.Samples, cloneSample(sample))
|
||||
}
|
||||
current.OmittedSampleCount = current.OccurrenceCount - len(current.Samples)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Diagnostics returns an independently owned snapshot in first-occurrence
|
||||
// order.
|
||||
func (collector *Collector) Diagnostics() []contracts.ProducerDiagnostic {
|
||||
if collector == nil {
|
||||
return nil
|
||||
}
|
||||
return contracts.CloneProducerDiagnostics(collector.diagnostics)
|
||||
}
|
||||
|
||||
func containsSample(samples []contracts.DiagnosticSample, candidate contracts.DiagnosticSample) bool {
|
||||
for _, sample := range samples {
|
||||
if sample.Scope == candidate.Scope && sample.Message == candidate.Message {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cloneSample(sample contracts.DiagnosticSample) contracts.DiagnosticSample {
|
||||
if sample.ChunkIndex != nil {
|
||||
chunkIndex := *sample.ChunkIndex
|
||||
sample.ChunkIndex = &chunkIndex
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
func maximumInt() int {
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
|
||||
type key struct {
|
||||
disposition contracts.DiagnosticDisposition
|
||||
category contracts.DiagnosticCategory
|
||||
reasonCode string
|
||||
}
|
||||
103
internal/framework/diagnostics/collector_test.go
Normal file
103
internal/framework/diagnostics/collector_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestCollectorCountsOccurrencesAndRetainsDistinctSamplesInOrder(t *testing.T) {
|
||||
collector := NewCollector()
|
||||
for _, diagnostic := range []contracts.ProducerDiagnostic{
|
||||
advisory("one", "first"),
|
||||
advisory("one", "first"),
|
||||
advisory("two", "second"),
|
||||
advisory("three", "third"),
|
||||
advisory("four", "fourth"),
|
||||
} {
|
||||
if err := collector.Add(diagnostic); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
diagnostics := collector.Diagnostics()
|
||||
if len(diagnostics) != 1 {
|
||||
t.Fatalf("group count = %d, want 1", len(diagnostics))
|
||||
}
|
||||
group := diagnostics[0]
|
||||
if group.OccurrenceCount != 5 || group.OmittedSampleCount != 2 {
|
||||
t.Fatalf("group counts = %#v, want five occurrences and two omitted samples", group)
|
||||
}
|
||||
if got := []string{group.Samples[0].Scope, group.Samples[1].Scope, group.Samples[2].Scope}; !equalStrings(got, []string{"one", "two", "three"}) {
|
||||
t.Fatalf("sample order = %#v, want first three distinct samples", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorSeparatesGroupsAndRejectsInvalidOrExcessiveGroups(t *testing.T) {
|
||||
collector := NewCollector()
|
||||
if err := collector.Add(advisory("one", "first")); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
warning := advisory("two", "second")
|
||||
warning.Disposition = contracts.DiagnosticDispositionWarning
|
||||
warning.Category = contracts.DiagnosticCategoryFallback
|
||||
if err := collector.Add(warning); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
if got := len(collector.Diagnostics()); got != 2 {
|
||||
t.Fatalf("group count = %d, want 2", got)
|
||||
}
|
||||
|
||||
invalid := advisory("bad", "bad")
|
||||
invalid.ReasonCode = ""
|
||||
if err := collector.Add(invalid); err == nil {
|
||||
t.Fatal("Add() error = nil, want invalid diagnostic error")
|
||||
}
|
||||
|
||||
limited := NewCollector()
|
||||
for index := 0; index < contracts.MaxProducerDiagnosticGroups; index++ {
|
||||
diagnostic := advisory("scope", "message")
|
||||
diagnostic.ReasonCode = "reason-" + string(rune('a'+index))
|
||||
if err := limited.Add(diagnostic); err != nil {
|
||||
t.Fatalf("Add(%d) error = %v", index, err)
|
||||
}
|
||||
}
|
||||
if err := limited.Add(advisory("overflow", "overflow")); err == nil {
|
||||
t.Fatal("Add() error = nil, want local group limit error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorReturnsIndependentSnapshots(t *testing.T) {
|
||||
collector := NewCollector()
|
||||
if err := collector.Add(advisory("scope", "message")); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
first := collector.Diagnostics()
|
||||
first[0].Samples[0].Message = "changed"
|
||||
second := collector.Diagnostics()
|
||||
if second[0].Samples[0].Message != "message" {
|
||||
t.Fatalf("collector snapshot changed = %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func advisory(scope string, message string) contracts.ProducerDiagnostic {
|
||||
return contracts.ProducerDiagnostic{
|
||||
Disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
Category: contracts.DiagnosticCategoryDataQuality,
|
||||
ReasonCode: "source_unrelated",
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}},
|
||||
}
|
||||
}
|
||||
|
||||
func equalStrings(left []string, right []string) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for index := range left {
|
||||
if left[index] != right[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -119,6 +119,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
|
||||
}
|
||||
appendedMessages, err := promptKitCorrectionMessages(req.Correction)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion correction: %w", err)
|
||||
}
|
||||
promptID := strings.TrimSpace(req.PromptID)
|
||||
if promptID == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||
@@ -133,13 +137,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
}
|
||||
|
||||
runReq := promptkit.RunRequest{
|
||||
PromptID: promptID,
|
||||
PromptVersion: strings.TrimSpace(req.PromptVersion),
|
||||
ProfileID: strings.TrimSpace(req.ProfileID),
|
||||
SessionID: sessionID,
|
||||
Inputs: promptKitInputs(req.Inputs),
|
||||
Vars: promptKitVars(req, sessionID),
|
||||
Execution: execution,
|
||||
PromptID: promptID,
|
||||
PromptVersion: strings.TrimSpace(req.PromptVersion),
|
||||
ProfileID: strings.TrimSpace(req.ProfileID),
|
||||
SessionID: sessionID,
|
||||
Inputs: promptKitInputs(req.Inputs),
|
||||
Vars: promptKitVars(req, sessionID),
|
||||
Execution: execution,
|
||||
AppendedMessages: appendedMessages,
|
||||
}
|
||||
if req.StructuredOutputRepairAttempts != nil {
|
||||
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
|
||||
@@ -213,6 +218,20 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
|
||||
owned, err := contracts.CloneSemanticCorrection(correction)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if owned == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: string(owned.AssistantResponse)},
|
||||
{Role: promptkit.RoleUser, Content: owned.UserGuidance},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
|
||||
if generationErr == nil {
|
||||
return nil
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -116,6 +117,74 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientAppendsSemanticCorrectionAfterRenderedPrompt(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
request := contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.direct-session",
|
||||
ProfileID: "explicit-profile",
|
||||
SessionID: "correction-session",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
}
|
||||
|
||||
var ordinary map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), request, &ordinary); err != nil {
|
||||
t.Fatalf("ordinary CompleteStructured() error = %v", err)
|
||||
}
|
||||
ordinaryMessages := append([]promptkit.RenderedMessage(nil), fake.lastRequest().Prompt.Messages...)
|
||||
if len(ordinaryMessages) != 1 {
|
||||
t.Fatalf("ordinary rendered messages = %#v, want only the declared prompt message", ordinaryMessages)
|
||||
}
|
||||
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"previous":"response"}`), "Return the corrected JSON object.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
request.Correction = correction
|
||||
var corrected map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), request, &corrected); err != nil {
|
||||
t.Fatalf("corrected CompleteStructured() error = %v", err)
|
||||
}
|
||||
correctedMessages := fake.lastRequest().Prompt.Messages
|
||||
if len(correctedMessages) != len(ordinaryMessages)+2 {
|
||||
t.Fatalf("corrected message count = %d, want %d", len(correctedMessages), len(ordinaryMessages)+2)
|
||||
}
|
||||
if !reflect.DeepEqual(correctedMessages[:len(ordinaryMessages)], ordinaryMessages) {
|
||||
t.Fatalf("ordinary rendered prefix changed: got %#v, want %#v", correctedMessages[:len(ordinaryMessages)], ordinaryMessages)
|
||||
}
|
||||
if got, want := correctedMessages[len(ordinaryMessages)], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"previous":"response"}`}); got != want {
|
||||
t.Fatalf("assistant correction message = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := correctedMessages[len(ordinaryMessages)+1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the corrected JSON object."}); got != want {
|
||||
t.Fatalf("user correction message = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *testing.T) {
|
||||
const sensitiveResponse = "assistant-response-must-not-appear-in-errors"
|
||||
for _, correction := range []*contracts.SemanticCorrection{
|
||||
{AssistantResponse: []byte(sensitiveResponse), UserGuidance: " \t"},
|
||||
{AssistantResponse: bytes.Repeat([]byte(sensitiveResponse), contracts.MaxAssistantResponseBytes/len(sensitiveResponse)+1), UserGuidance: "Use a smaller response."},
|
||||
} {
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{Correction: correction}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "correction") {
|
||||
t.Fatalf("CompleteStructured() error = %v, want correction validation failure", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), sensitiveResponse) {
|
||||
t.Fatalf("correction validation error leaked response content: %v", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 0 {
|
||||
t.Fatalf("provider calls = %d, want no provider call after invalid correction", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
|
||||
const initialPrompt = `id: snapshot.test
|
||||
version: "v1"
|
||||
@@ -960,6 +1029,10 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
|
||||
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
|
||||
}}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"bad":true}`), "Return the required ok field.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
|
||||
var out struct {
|
||||
OK bool `json:"ok"`
|
||||
@@ -968,6 +1041,7 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
|
||||
PromptID: "adapter.test",
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
SessionID: "repair-test",
|
||||
Correction: correction,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
@@ -978,6 +1052,17 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 2 {
|
||||
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
|
||||
}
|
||||
requests := fake.requestsSnapshot()
|
||||
if len(requests) != 2 || len(requests[0].Prompt.Messages) < 3 {
|
||||
t.Fatalf("repair requests = %#v, want correction messages on the initial prepared request", requests)
|
||||
}
|
||||
messages := requests[0].Prompt.Messages
|
||||
if got, want := messages[len(messages)-2], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"bad":true}`}); got != want {
|
||||
t.Fatalf("repair assistant correction = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := messages[len(messages)-1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the required ok field."}); got != want {
|
||||
t.Fatalf("repair user correction = %#v, want %#v", got, want)
|
||||
}
|
||||
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
|
||||
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ const (
|
||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||
// release. Update this identity when the dependency is upgraded.
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.8.0:builtin-profiles"
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.9.0:builtin-profiles"
|
||||
)
|
||||
|
||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||
|
||||
@@ -28,14 +28,14 @@ type CheckpointRecorder interface {
|
||||
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
|
||||
SourceFailed(moduleKey string, err error) error
|
||||
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
|
||||
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
|
||||
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
|
||||
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
}
|
||||
@@ -45,14 +45,14 @@ type CheckpointRecorder interface {
|
||||
// remain available for callers that do not have step context.
|
||||
type StepCheckpointRecorder interface {
|
||||
ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
|
||||
ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
|
||||
MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
|
||||
NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
}
|
||||
@@ -89,6 +89,7 @@ const (
|
||||
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
|
||||
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
|
||||
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
|
||||
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
|
||||
)
|
||||
|
||||
type CheckpointDecision struct {
|
||||
@@ -160,6 +161,8 @@ func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
|
||||
return "accepted normalized artifact is reusable"
|
||||
case CheckpointReasonRecomputeStep:
|
||||
return "selected step requires execution"
|
||||
case CheckpointReasonValidationIncompleteLineage:
|
||||
return "checkpoint reuse is disabled by validation-incomplete input lineage"
|
||||
default:
|
||||
return "checkpoint decision"
|
||||
}
|
||||
@@ -276,21 +279,27 @@ type CheckpointArtifact struct {
|
||||
ChunkRef source.SourceRef
|
||||
Artifact contracts.SerializedArtifact
|
||||
SchemaDigest string
|
||||
Diagnostics []CheckpointDiagnostic
|
||||
}
|
||||
|
||||
// CheckpointDiagnostic stores a producer-local diagnostic alongside a
|
||||
// reusable artifact. The runner supplies the current run's origin when it
|
||||
// promotes this value into a diagnostic group.
|
||||
type CheckpointDiagnostic struct {
|
||||
Diagnostic contracts.ProducerDiagnostic `json:"diagnostic"`
|
||||
ValidatorKey string `json:"validator_key,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractCheckpoint struct {
|
||||
Outputs []CheckpointArtifact
|
||||
Rejected []contracts.RejectedOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type MergeCheckpoint struct {
|
||||
Output CheckpointArtifact
|
||||
Warnings []contracts.Warning
|
||||
Output CheckpointArtifact
|
||||
}
|
||||
type NormalizeCheckpoint struct {
|
||||
Output CheckpointArtifact
|
||||
Warnings []contracts.Warning
|
||||
Output CheckpointArtifact
|
||||
}
|
||||
|
||||
type CheckpointLoader interface {
|
||||
@@ -323,14 +332,14 @@ func (noopCheckpointRecorder) SourceFailed(string, error) error
|
||||
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
@@ -342,7 +351,7 @@ func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprin
|
||||
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
@@ -375,11 +384,11 @@ func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, modul
|
||||
}
|
||||
return recorder.ExtractRunning(laneID, moduleKey, deps)
|
||||
}
|
||||
func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected, warnings)
|
||||
return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected)
|
||||
}
|
||||
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
|
||||
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected)
|
||||
}
|
||||
func checkpointExtractFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
@@ -393,11 +402,11 @@ func checkpointMergeRunning(recorder CheckpointRecorder, stepID, laneID, moduleK
|
||||
}
|
||||
return recorder.MergeRunning(laneID, moduleKey, deps)
|
||||
}
|
||||
func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings)
|
||||
return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output)
|
||||
}
|
||||
return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
return recorder.MergeSucceeded(laneID, moduleKey, deps, output)
|
||||
}
|
||||
func checkpointMergeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
@@ -417,11 +426,11 @@ func checkpointNormalizeRunning(recorder CheckpointRecorder, stepID, laneID, mod
|
||||
}
|
||||
return recorder.NormalizeRunning(laneID, moduleKey, deps)
|
||||
}
|
||||
func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings)
|
||||
return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output)
|
||||
}
|
||||
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output)
|
||||
}
|
||||
func checkpointNormalizeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const ChunkPlanSchemaVersion = "notarius.chunk-plan.v2"
|
||||
const ChunkPlanSchemaVersion = "notarius.chunk-plan.v3"
|
||||
|
||||
type ChunkPlanProducer struct {
|
||||
InputModule string `json:"input_module"`
|
||||
@@ -19,13 +19,13 @@ type ChunkPlanProducer struct {
|
||||
}
|
||||
|
||||
type ChunkPlanRecord struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
SourceDigest string `json:"source_digest"`
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
Plan source.ChunkPlan `json:"plan"`
|
||||
Producer ChunkPlanProducer `json:"producer"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
SourceDigest string `json:"source_digest"`
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
Plan source.ChunkPlan `json:"plan"`
|
||||
Producer ChunkPlanProducer `json:"producer"`
|
||||
Diagnostics []contracts.ProducerDiagnostic `json:"diagnostics,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ChunkPlanStore interface {
|
||||
|
||||
@@ -42,6 +42,7 @@ type debugTimedEnvelope struct {
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
AttemptKind string `json:"attempt_kind,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
@@ -51,11 +52,10 @@ type debugTimedEnvelope struct {
|
||||
}
|
||||
|
||||
type debugBinaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugSourceInput struct {
|
||||
@@ -110,24 +110,7 @@ type debugSerializedOutput struct {
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
type debugLLMInputMaterial struct {
|
||||
Name string `json:"name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Content string `json:"content_base64,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
}
|
||||
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
|
||||
|
||||
type debugStructuredCompletionResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
@@ -160,9 +143,37 @@ type debugLLMCallReference struct {
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||
Error bool `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationOutcome struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Outcome string `json:"outcome"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
}
|
||||
|
||||
type debugProducerAttempt struct {
|
||||
Number int `json:"number"`
|
||||
Kind string `json:"kind"`
|
||||
Outcome string `json:"outcome"`
|
||||
Validation []debugValidationOutcome `json:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type debugProducerTerminal struct {
|
||||
ProducerAttemptCount int `json:"producer_attempt_count"`
|
||||
Attempts []debugProducerAttempt `json:"attempts,omitempty"`
|
||||
AggregateReasonCodes []string `json:"aggregate_reason_codes,omitempty"`
|
||||
ValidationComplete bool `json:"validation_complete"`
|
||||
EffectivePolicy ValidationPolicy `json:"effective_policy"`
|
||||
TerminalAction string `json:"terminal_action"`
|
||||
Summary artifacts.ValidationSummary `json:"validation_summary"`
|
||||
}
|
||||
|
||||
type debugValidationCall struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Request any `json:"request"`
|
||||
@@ -267,6 +278,10 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
|
||||
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
|
||||
PromptTokens: response.PromptTokens,
|
||||
CompletionTokens: response.CompletionTokens,
|
||||
TotalTokens: response.TotalTokens,
|
||||
RepairAttempts: response.RepairAttempts,
|
||||
Error: err != nil,
|
||||
}
|
||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||
@@ -423,14 +438,57 @@ func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
|
||||
return terminalErr
|
||||
}
|
||||
|
||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
|
||||
func debugValidationOutcomes(report validationReport) []debugValidationOutcome {
|
||||
if len(report.records) == 0 {
|
||||
return nil
|
||||
}
|
||||
outcomes := make([]debugValidationOutcome, 0, len(report.records))
|
||||
for _, record := range report.records {
|
||||
outcomes = append(outcomes, debugValidationOutcome{
|
||||
ValidatorName: record.validatorName,
|
||||
Outcome: string(record.outcome),
|
||||
AttemptCount: record.attemptCount,
|
||||
ReasonCode: record.reasonCode,
|
||||
})
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
|
||||
func writeProducerTerminalDebug(recorder DebugRecorder, name string, terminal producerAttemptTerminal, policy ValidationPolicy, summary artifacts.ValidationSummary) error {
|
||||
attempts := make([]debugProducerAttempt, 0, len(terminal.Provenance))
|
||||
for _, item := range terminal.Provenance {
|
||||
attempts = append(attempts, debugProducerAttempt{
|
||||
Number: item.Number,
|
||||
Kind: string(item.Kind),
|
||||
Outcome: string(item.Outcome),
|
||||
Validation: debugValidationOutcomes(item.Validation),
|
||||
})
|
||||
}
|
||||
return writeDebugTimed(recorder, name, debugTimedEnvelope{
|
||||
Stage: summary.Stage,
|
||||
StepID: summary.StepID,
|
||||
LaneID: summary.LaneID,
|
||||
ModuleKey: summary.ModuleKey,
|
||||
StartedAt: time.Now().UTC(),
|
||||
Payload: debugProducerTerminal{
|
||||
ProducerAttemptCount: summary.ProducerAttemptCount,
|
||||
Attempts: attempts,
|
||||
AggregateReasonCodes: append([]string(nil), summary.ReasonCodes...),
|
||||
ValidationComplete: summary.Status == "complete",
|
||||
EffectivePolicy: policy,
|
||||
TerminalAction: string(terminal.Action),
|
||||
Summary: artifacts.CloneValidationSummary(summary),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
|
||||
content = redactSecretBytes(content)
|
||||
return debugBinaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: debugContentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: redactSensitiveMap(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -567,29 +625,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
||||
}
|
||||
|
||||
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
||||
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
|
||||
for key, material := range req.Inputs {
|
||||
inputs[key] = debugLLMInputMaterial{
|
||||
Name: material.Name,
|
||||
MediaType: material.MediaType,
|
||||
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
|
||||
Digest: material.Digest,
|
||||
OriginURI: material.OriginURI,
|
||||
SizeBytes: material.SizeBytes,
|
||||
}
|
||||
}
|
||||
if len(inputs) == 0 {
|
||||
inputs = nil
|
||||
}
|
||||
return debugStructuredCompletionRequest{
|
||||
StageName: req.StageName,
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
Vars: redactSensitiveMap(req.Vars),
|
||||
}
|
||||
return req.DebugSummary()
|
||||
}
|
||||
|
||||
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
||||
@@ -691,21 +727,11 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
|
||||
|
||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||
result.CorrectionGuidance = ""
|
||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||
for i := range result.Warnings {
|
||||
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func debugWarningEnvelopes(warnings []contracts.Warning) []contracts.Warning {
|
||||
out := cloneWarnings(warnings)
|
||||
for i := range out {
|
||||
out[i].Message = string(redactSecretBytes([]byte(out[i].Message)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugRejectedOutputEnvelope(rejected contracts.RejectedOutput) contracts.RejectedOutput {
|
||||
rejected.Message = string(redactSecretBytes([]byte(rejected.Message)))
|
||||
rejected.DiagnosticArtifactPath = string(redactSecretBytes([]byte(rejected.DiagnosticArtifactPath)))
|
||||
|
||||
@@ -47,6 +47,32 @@ func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) {
|
||||
const assistantResponse = `{"secret":"assistant response"}`
|
||||
const userGuidance = "secret user guidance"
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
summary := debugCompletionRequest(contracts.StructuredCompletionRequest{
|
||||
Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
Correction: correction,
|
||||
})
|
||||
encoded, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal completion summary: %v", err)
|
||||
}
|
||||
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||
if strings.Contains(string(encoded), secret) {
|
||||
t.Fatalf("debug completion summary leaked %q: %s", secret, encoded)
|
||||
}
|
||||
}
|
||||
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||
t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
|
||||
doc := validSourceDocument()
|
||||
envelope := debugSourceDocumentEnvelope(doc)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user