Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 77ae132e56 | |||
| deebc89255 | |||
| f208dbe954 | |||
| 7e626753bf | |||
| 917d150279 | |||
| 4da9360d74 | |||
| 5cab4e512e | |||
| b178f1c684 | |||
| a2610757cd | |||
| 3ad34956c5 | |||
| 22e6caa2a0 | |||
| 9cb7462800 | |||
| 87385b7e14 | |||
| 0ae5ea7637 | |||
| 6bc883dfb6 | |||
| fcfff3ad15 | |||
| 0f2b23dce1 |
@@ -16,7 +16,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"turn_kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["turn", "reaction", "legendary_action", "lair_action", "other"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
"required": ["name", "kind", "source_refs"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"kind": {"type": "string"},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": ["engaged", "killed", "fled", "captured", "incapacitated"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
||||
@@ -18,12 +18,20 @@ when the transcript explicitly describes it being physically destroyed or
|
||||
expended as a non-payment component. Use `transferred` only when possession
|
||||
moves between two distinct named party members.
|
||||
|
||||
Return both `from` and `to` for every occurrence, using `null` when a holder does not
|
||||
apply. For `discovered`, set both holders to `null`. For `acquired`, set `from`
|
||||
to `null` and provide `to`; for `lost` and `consumed`, provide `from` and set
|
||||
`to` to `null`; and for `transferred`, provide both holders. Use `party` only
|
||||
for collective or unresolved party possession, never for either side of a
|
||||
transfer. Do not emit a transfer for a gift, sale, or payment outside the party.
|
||||
Return both `from` and `to` for every occurrence. Use JSON `null`, not an empty
|
||||
string, whenever a holder does not apply. Follow this holder matrix exactly:
|
||||
|
||||
| `kind` | required `from` | required `to` |
|
||||
| --- | --- | --- |
|
||||
| `discovered` | `null` | `null` |
|
||||
| `acquired` | `null` | `party` or the named party member gaining possession |
|
||||
| `lost` | `party` or the named party member losing possession | `null` |
|
||||
| `consumed` | `party` or the named party member consuming the item | `null` |
|
||||
| `transferred` | one named party member | a different named party member |
|
||||
|
||||
Use `party` only for collective or unresolved party possession, never for
|
||||
either side of a transfer. Do not emit a transfer for a gift, sale, or payment
|
||||
outside the party.
|
||||
|
||||
Ordinary non-depleting use is not an occurrence. Do not infer acquisition from a
|
||||
discovery, or discovery from an acquisition: emit both only when each is
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
"required": ["name", "kind", "quantity", "from", "to", "source_refs"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"kind": {"type": "string"},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": ["discovered", "acquired", "lost", "consumed", "transferred"]
|
||||
},
|
||||
"quantity": {"type": ["integer", "null"]},
|
||||
"from": {"type": ["string", "null"]},
|
||||
"to": {"type": ["string", "null"]},
|
||||
|
||||
@@ -5,5 +5,5 @@ evidence, similar objects, or a shared owner as sufficient.
|
||||
Keep currency denominations and materially different item types separate. Keep
|
||||
uncertain aliases separate. Do not infer an item property or uniqueness.
|
||||
|
||||
When selecting a canonical display name, choose one supplied candidate name
|
||||
that is the clearest established designation.
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the clearest established designation.
|
||||
|
||||
@@ -5,4 +5,5 @@ nearby evidence, nested places, or generic labels as sufficient.
|
||||
Keep parent and child places separate, as well as similarly named places and
|
||||
uncertain aliases.
|
||||
|
||||
When selecting a canonical display name, prefer the clearest established name.
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the clearest established name.
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"]
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
|
||||
@@ -3,7 +3,8 @@ contextual labels and cited transcript windows. Preserve distinct individuals
|
||||
even when their names are similar or their contextual descriptions are
|
||||
identical.
|
||||
|
||||
When selecting a canonical display name, prefer a complete, stable proper name
|
||||
Set `canonical_candidate_number` to the supplied candidate number whose label
|
||||
is the preferred canonical display name. Prefer a complete, stable proper name
|
||||
over an abbreviation. Prefer an unadorned proper name over that name plus a
|
||||
contextual class, role, title, or relationship descriptor unless the transcript
|
||||
establishes the descriptor as part of the person's name. A longer display name
|
||||
|
||||
@@ -5,9 +5,7 @@ into multiple scenes or use facts that are not supported by it.
|
||||
Return one kind, one concise title, and one concise summary. Choose exactly one
|
||||
kind:
|
||||
|
||||
- combat: active combat materially organizes the scene, including
|
||||
initiative-like exchanges or sustained hostile action. Planning a fight or
|
||||
discussing a completed fight is not combat by itself.
|
||||
- combat: a scene classified as combat under the shared combat policy.
|
||||
- narrative: current-session in-world play that is not principally active
|
||||
combat, a prior-session recap, or sustained out-of-character session
|
||||
discussion. This includes exploration, travel, dialogue, investigation,
|
||||
@@ -20,15 +18,13 @@ kind:
|
||||
play.
|
||||
|
||||
Narrative is the default for actual current-session gameplay that does not meet
|
||||
another definition. When the accepted chunk is mixed:
|
||||
another definition. When the accepted chunk has no substantive active combat:
|
||||
|
||||
1. use combat when active combat is a substantive central activity, even with
|
||||
brief setup, rules clarification, or immediate aftermath;
|
||||
2. otherwise use recap when recounting a previous session is the chunk's
|
||||
1. use recap when recounting a previous session is the chunk's
|
||||
primary table purpose;
|
||||
3. otherwise use meta when sustained out-of-character session discussion is
|
||||
2. otherwise use meta when sustained out-of-character session discussion is
|
||||
primary and in-world progression is no more than incidental; and
|
||||
4. use narrative for all remaining current-session in-world play.
|
||||
3. use narrative for all remaining current-session in-world play.
|
||||
|
||||
Brief table talk, dice resolution, rules clarification, jokes, or
|
||||
administrative comments do not make a gameplay scene meta. A short recollection
|
||||
|
||||
@@ -27,6 +27,8 @@ messages:
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
"required": ["kind", "title", "summary"],
|
||||
"properties": {
|
||||
"kind": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"enum": ["combat", "narrative", "recap", "meta"]
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
Classify only whether substantive active combat occurs in the supplied
|
||||
transcript chunk under the shared combat policy. Do not judge the scene title,
|
||||
summary, non-combat subtype, scene boundary, or any other aspect of a scene
|
||||
description.
|
||||
|
||||
Return `combat` when the chunk contains substantive active combat and
|
||||
`non_combat` otherwise. Give a concise, transcript-grounded explanation for
|
||||
the classification.
|
||||
@@ -0,0 +1,23 @@
|
||||
id: dnd.scene_descriptions.validate_combat
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||
repair_attempts: 1
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.scene_descriptions.combat_semantics.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["classification", "explanation"],
|
||||
"properties": {
|
||||
"classification": {
|
||||
"type": "string",
|
||||
"enum": ["combat", "non_combat"]
|
||||
},
|
||||
"explanation": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 512
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
Transcript units are the only evidence for extracted events and factual claims.
|
||||
Every reported factual claim must be supported by cited transcript units. Use
|
||||
integer `start_unit_id` and `end_unit_id` values from the transcript.
|
||||
Within each range, `start_unit_id` must identify the earlier transcript unit and
|
||||
`end_unit_id` the same or a later unit according to transcript order.
|
||||
|
||||
When supporting evidence is non-contiguous, use multiple narrow ranges rather
|
||||
than a broad range that bridges unrelated conversation.
|
||||
|
||||
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
@@ -0,0 +1,13 @@
|
||||
Use `combat` only when substantive active combat materially organizes the
|
||||
scene. Active combat includes initiative or turn exchanges, attacks, combat
|
||||
spells, damage, saves, movement, or similarly sustained hostile action.
|
||||
|
||||
Do not use `combat` for planning or preparing for a possible fight; threats,
|
||||
hostile dialogue, or a tense confrontation; immediate aftermath, looting,
|
||||
healing, or discussion of a completed fight; a recap or in-world recollection
|
||||
of earlier combat; or out-of-character rules discussion without active
|
||||
encounter play.
|
||||
|
||||
When a chunk contains substantive active combat alongside brief setup, rules
|
||||
clarification, interruption, phase transition, or immediate aftermath, classify
|
||||
it as `combat`.
|
||||
@@ -1,3 +1,3 @@
|
||||
Candidate material:
|
||||
Candidate material, including the exact valid candidate-number range:
|
||||
|
||||
{{ input "candidates" }}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
Use only the positive integer `candidate_id` values supplied in the candidate material.
|
||||
Use only the positive integer `candidate_number` values supplied in the candidate material. Valid candidate numbers are exactly the inclusive `first` through `last` values declared in `candidate_number_range`; use the explicit number attached to each candidate.
|
||||
|
||||
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate IDs, and its `canonical_candidate_id` must be one of those IDs. A candidate may appear in at most one group.
|
||||
Transcript unit `id` values and evidence `start_unit_id` and `end_unit_id` values identify source positions. They are not candidate numbers and must never be used in `candidate_numbers` or `canonical_candidate_number`.
|
||||
|
||||
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer an ID from list position. An empty `duplicate_groups` array is valid.
|
||||
Return a duplicate group only when the evidence supports that every selected candidate describes the same underlying entity. Each group must contain at least two distinct candidate numbers, and its `canonical_candidate_number` must be one of those numbers. A candidate may appear in at most one group.
|
||||
|
||||
The response must conform exactly to the selected JSON schema. Return IDs only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||
Omit uncertain matches and candidates that should remain distinct. Do not invent candidates or infer a number that is not explicitly supplied. An empty `duplicate_groups` array is valid.
|
||||
|
||||
The response must conform exactly to the selected JSON schema. Return candidate numbers only: do not copy candidate names, evidence, transcript text, source identifiers, or source ranges into the response.
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["candidate_ids", "canonical_candidate_id"],
|
||||
"required": ["candidate_numbers", "canonical_candidate_number"],
|
||||
"properties": {
|
||||
"candidate_ids": {
|
||||
"candidate_numbers": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"items": {
|
||||
@@ -21,7 +21,7 @@
|
||||
"minimum": 1
|
||||
}
|
||||
},
|
||||
"canonical_candidate_id": {
|
||||
"canonical_candidate_number": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
|
||||
76
docs/adr/0016-feedback-aware-module-requested-retries.md
Normal file
76
docs/adr/0016-feedback-aware-module-requested-retries.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# ADR-0016: Use feedback-aware module-requested retries
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-08-28
|
||||
|
||||
## Context
|
||||
|
||||
An LLM-backed module can discover that a structurally valid model response is
|
||||
unsafe while translating it into a typed candidate, before the ordinary
|
||||
validator chain runs. Semantic registry reconciliation is the first such case:
|
||||
the shared core can identify invalid duplicate-group proposals, and typed
|
||||
application can reject a domain-incompatible group while preserving a safe
|
||||
partial result. Repeating the original request without the rejected response or
|
||||
corrective explanation gives the model no information with which to improve.
|
||||
|
||||
The existing feedback-aware validation mechanism already establishes the safe
|
||||
correction protocol, but module-owned retry directives also carry internal
|
||||
reason codes, operator messages, and fallback diagnostics. Those values are not
|
||||
model instructions. Some module retry conditions, including exhausted
|
||||
structured-output repair, also have no valid exact response to append.
|
||||
|
||||
## Decision
|
||||
|
||||
An LLM-backed normalizer may attach optional, bounded model-facing correction
|
||||
guidance to a module-requested retry. Guidance is a separate contract field; the
|
||||
framework never derives it from a reason code, operator message, diagnostic, or
|
||||
error. A feedback-capable directive must include the exact model candidate that
|
||||
controlled the safe fallback and must use `single_response_v1`.
|
||||
|
||||
The artifact-neutral producer-attempt state machine constructs the correction
|
||||
from that exact latest response and the supplied guidance. The producer rebuilds
|
||||
its complete ordinary request and appends the response as one assistant message
|
||||
and the guidance as one user message. Earlier attempts do not accumulate, and
|
||||
the attempt consumes the same configured stage retry budget as operational,
|
||||
structural, validation, and feedback-free module retries.
|
||||
|
||||
A module retry without guidance remains valid and starts a fresh attempt. This
|
||||
is the required behavior when no valid model candidate exists, including after
|
||||
structured-output repair is exhausted. If feedback is supplied without a valid
|
||||
supported candidate, the framework fails the module contract rather than
|
||||
retrying blindly or inventing guidance.
|
||||
|
||||
After retry exhaustion, the normalizer's safe fallback continues through its
|
||||
validator chain. Safe semantic groups may remain applied, unsafe groups remain
|
||||
separate, and bounded fallback diagnostics may explain the process degradation.
|
||||
Exact responses and correction text remain attempt-local and are excluded from
|
||||
ordinary errors, warnings, manifests, receipts, caches, and checkpoints.
|
||||
|
||||
This decision extends, rather than supersedes,
|
||||
[ADR-0014](0014-feedback-aware-validation-retries.md): both correction sources
|
||||
use the same transport-neutral payload, replacement-request semantics, outer
|
||||
retry budget, and sensitive-data boundary.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Continue blind module retries. This preserves a smaller contract but wastes
|
||||
the module's deterministic diagnosis and commonly repeats the same defect.
|
||||
- Convert module safety checks into validators. Typed reconciliation must apply
|
||||
only safe proposal groups and retain a fallback before validation; moving
|
||||
artifact-owned translation and application policy into validators would blur
|
||||
stage ownership.
|
||||
- Copy the retry reason or operator message into the model request. Those values
|
||||
are written for provenance and humans, can contain opaque internal labels,
|
||||
and do not reliably describe a correct replacement.
|
||||
- Require feedback for every module retry. Structural failures may have no
|
||||
valid exact candidate, so this would either prevent useful fresh retries or
|
||||
fabricate prior-response material.
|
||||
|
||||
## Consequences
|
||||
|
||||
The normalize retry contract and generic producer-attempt directive gain an
|
||||
optional correction-guidance field and candidate-pairing validation. Modules
|
||||
that use it must provide semantically meaningful bounded prose and an exact
|
||||
candidate. Registry reconciliation maintains separate operator and model
|
||||
renderers, and policy fingerprints change so checkpoints created under blind
|
||||
retry behavior are not reused.
|
||||
44
docs/cli.md
44
docs/cli.md
@@ -55,8 +55,8 @@ pipeline ID and **--input** are required.
|
||||
| **--session-id id** | Override the generated prompt session identifier with a non-empty value for LLM-backed module calls. |
|
||||
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
|
||||
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
|
||||
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. |
|
||||
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. |
|
||||
| **--reference selector=path** | Add or replace external file reference bindings at pipeline, lane, chunk, or binding scope. Repeatable. |
|
||||
| **--without-reference selector** | Remove matching configured external reference bindings. Repeatable. |
|
||||
|
||||
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
|
||||
**--debug-dir**, **--output-dir**, **--session-id**, and
|
||||
@@ -81,23 +81,49 @@ guidance.
|
||||
|
||||
### Reference selectors
|
||||
|
||||
Use **--reference** only for a reference slot declared by the selected
|
||||
configured target. The accepted selector forms are:
|
||||
Use **--reference** only for reference slots declared by the selected
|
||||
configured targets. Qualification narrows the scope of an override:
|
||||
|
||||
| Form | Target |
|
||||
| --- | --- |
|
||||
| slot=path | The unique selected target that declares slot. |
|
||||
| slot=path | Every selected target that declares slot. |
|
||||
| chunk.slot=path | The chunker. |
|
||||
| merge.slot=path | The unique selected merger that declares slot. |
|
||||
| lane.slot=path | The unique extractor, merger, or normalizer in lane that declares slot. |
|
||||
| lane.slot=path | Every extractor, merger, or normalizer in lane that declares slot. |
|
||||
| lane.extract.slot=path | The extractor in lane. |
|
||||
| lane.merge.slot=path | The merger in lane. |
|
||||
| lane.normalize.slot=path | The normalizer in lane. |
|
||||
|
||||
**--without-reference** uses the same selector forms without =path. Slot
|
||||
names, requiredness, and configured bindings are part of the
|
||||
Pipeline- and lane-scoped selectors are expected to match multiple targets and
|
||||
fail if they match none. A stage-specific selector fails when its lane is not
|
||||
selected or its target does not declare the slot. There is no stage-wide
|
||||
`merge.slot` shorthand; name the lane when targeting a merger.
|
||||
|
||||
CLI bindings override configured external paths. For overlapping CLI
|
||||
selectors, a binding-specific or chunk selector overrides a lane selector, and
|
||||
a lane selector overrides a pipeline selector. The last occurrence wins at
|
||||
equal scope. Binding and unbinding the same concrete target at equal scope is
|
||||
an error; a narrower bind or unbind may create an intentional exception to a
|
||||
broader action.
|
||||
|
||||
**--without-reference** uses the same selector forms without `=path` and
|
||||
removes external bindings only. Neither flag replaces or removes a generated
|
||||
artifact handoff; an external/generated collision is a resolution error.
|
||||
Required slots are checked after all effective changes. CLI reference paths
|
||||
are resolved relative to the process working directory, so subprocess and
|
||||
service callers should use absolute paths. Slot names, accepted media types,
|
||||
size limits, requiredness, and configured generated bindings are part of the
|
||||
[configuration contract](config.md).
|
||||
|
||||
For example, one shared campaign reference can reach every compatible target,
|
||||
with an optional lane-specific exception:
|
||||
|
||||
~~~
|
||||
notarius run dnd-session \
|
||||
--input /data/transcript.json \
|
||||
--reference party=/data/references/party.txt \
|
||||
--reference npc-registry.party=/data/references/npc-party-context.txt
|
||||
~~~
|
||||
|
||||
### Run output
|
||||
|
||||
Without **--json**, standard output contains the completed pipeline ID, counts
|
||||
|
||||
@@ -411,10 +411,15 @@ slot. A generated binding supplies one accepted normalized artifact; it does
|
||||
not name a file. A configured generated dependency remains required even when
|
||||
that consumer slot is otherwise optional.
|
||||
|
||||
Pipeline references are defaults. A matching step-local or binding-local
|
||||
external path overrides a pipeline default. Required slots must be bound after
|
||||
these configuration values and any CLI reference overrides are applied.
|
||||
Reference paths in YAML are resolved relative to the configuration file.
|
||||
Pipeline references are configuration defaults. A matching step-local or
|
||||
binding-local external path overrides a pipeline default. CLI reference
|
||||
bindings are then operational overrides of configured external paths; their
|
||||
pipeline, lane, and binding scopes and precedence are defined by the
|
||||
[CLI reference](cli.md#reference-selectors). A CLI file reference cannot
|
||||
replace a configured generated artifact handoff. Required slots must be bound
|
||||
after configuration and CLI reference actions are applied. Reference paths in
|
||||
YAML are resolved relative to the configuration file; CLI reference paths are
|
||||
resolved relative to the process working directory.
|
||||
|
||||
### D&D Reference Slots
|
||||
|
||||
@@ -504,11 +509,33 @@ Available validator keys are:
|
||||
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
|
||||
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
|
||||
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
|
||||
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
|
||||
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
|
||||
|
||||
`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene:
|
||||
|
||||
~~~yaml
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 1
|
||||
validators:
|
||||
- generic/valid_json
|
||||
- extract/dnd/scene-descriptions/shape
|
||||
- extract/dnd/scene-descriptions/source_refs
|
||||
- generic/valid_json_schema
|
||||
- extract/dnd/scene-descriptions/source_relatedness
|
||||
- module: extract/dnd/scene-descriptions/combat_semantics
|
||||
retries: 1
|
||||
~~~
|
||||
|
||||
An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules.
|
||||
|
||||
The validator retry shown above permits one additional execution against the
|
||||
same scene candidate when the LLM-backed validator itself fails; it is separate
|
||||
from both the extractor's producer retry and PromptKit structural repair.
|
||||
|
||||
When no override is configured, production D&D bindings use the following
|
||||
ordered chains. Each row lists extract then normalize; spell chains are the
|
||||
same at both stages.
|
||||
|
||||
@@ -24,7 +24,9 @@ files. Use absolute paths for service and subprocess deployments. In
|
||||
particular, observe these different resolution rules:
|
||||
|
||||
- reference paths in YAML are resolved relative to the Notarius configuration
|
||||
file; and
|
||||
file;
|
||||
- reference paths passed with `--reference` are resolved relative to the
|
||||
Notarius process working directory; and
|
||||
- `promptkit.profile_file` is resolved relative to the Notarius process working
|
||||
directory.
|
||||
|
||||
@@ -73,9 +75,21 @@ notarius run dnd-session \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||
--output-dir /absolute/path/to/notarius-output \
|
||||
--reference party=/absolute/path/to/references/party.txt \
|
||||
--reference players=/absolute/path/to/references/players.txt \
|
||||
--reference glossary=/absolute/path/to/references/glossary.txt \
|
||||
--reference spell_catalog=/absolute/path/to/references/spells.json \
|
||||
--json
|
||||
```
|
||||
|
||||
Each unqualified reference is pipeline-scoped: Notarius supplies it to every
|
||||
selected D&D target that declares the slot. A deployment may omit an optional
|
||||
reference it does not maintain, and may use the lane- or binding-qualified
|
||||
forms from the [CLI reference](../cli.md#reference-selectors) for an exceptional
|
||||
override. The registry, scene-description, combat-turn, and NPC-occurrence
|
||||
references declared between ordered steps in the complete configuration are
|
||||
generated artifacts. Do not pass those handoffs on the CLI.
|
||||
|
||||
The caller should:
|
||||
|
||||
- capture stdout and stderr separately;
|
||||
|
||||
@@ -31,13 +31,36 @@ notarius run pipeline-id \
|
||||
```
|
||||
|
||||
Use absolute paths for supplied input, configuration, output-root, and
|
||||
reference files. Notarius generates a stable prompt session for the resolved
|
||||
input module and exact input bytes. Pass **--session-id** only when intentionally
|
||||
grouping different invocations under a different session. Supply credentials
|
||||
through Notarius's documented configuration and environment mechanisms, never
|
||||
as command-line arguments or generated secret-bearing configuration. In
|
||||
particular, a session identifier is provider-visible and is not a credential
|
||||
mechanism.
|
||||
reference files. Pass each external reference as its own argument-vector pair;
|
||||
do not construct and invoke a shell command. An unqualified reference selector
|
||||
supplies that file to every compatible selected target. Lane and stage
|
||||
qualification are available for exceptional overrides, while generated
|
||||
same-run references remain part of configured pipeline composition. The
|
||||
[CLI reference](../cli.md#reference-selectors) owns the exact selector and
|
||||
precedence contract.
|
||||
|
||||
The maintained D&D subprocess workflow uses this facility for campaign context:
|
||||
|
||||
```sh
|
||||
notarius run dnd-session \
|
||||
--config /absolute/path/to/notarius.yml \
|
||||
--input /absolute/path/to/transcripts/final.trimmed.json \
|
||||
--output-dir /absolute/path/to/notarius-output \
|
||||
--reference party=/absolute/path/to/references/party.txt \
|
||||
--reference players=/absolute/path/to/references/players.txt \
|
||||
--reference glossary=/absolute/path/to/references/glossary.txt \
|
||||
--reference spell_catalog=/absolute/path/to/references/spells.json \
|
||||
--json
|
||||
```
|
||||
|
||||
Only pass the external references available to and desired by the deployment.
|
||||
Notarius generates a stable prompt session for the resolved input module and
|
||||
exact input bytes; reference changes do not change it. Pass **--session-id**
|
||||
only when intentionally grouping different invocations under a different
|
||||
session. Supply credentials through Notarius's documented configuration and
|
||||
environment mechanisms, never as command-line arguments or generated
|
||||
secret-bearing configuration. In particular, a session identifier is
|
||||
provider-visible and is not a credential mechanism.
|
||||
|
||||
Wait for the process before interpreting standard output. Only an exit status
|
||||
of 0 permits decoding the receipt. On a nonzero exit, retain standard error for
|
||||
|
||||
@@ -115,6 +115,16 @@ to checkpoint identity and `pipeline.RunInput`. The public flag and stability
|
||||
contract are defined by the [CLI reference](../cli.md#run); framework and LLM
|
||||
packages only transport the supplied value.
|
||||
|
||||
The CLI also owns the scope grammar for reference flags. It enumerates the
|
||||
selected chunk and lane targets from registered module specifications, expands
|
||||
pipeline- and lane-scoped actions into exact stage-and-lane bindings, and
|
||||
resolves overlapping bind and unbind actions by specificity before calling
|
||||
configuration resolution. The generic pipeline therefore receives only exact
|
||||
`ReferenceBinding` and `ReferenceUnbind` values and has no knowledge of CLI
|
||||
selector syntax. Configuration resolution retains ownership of configured
|
||||
external/generated conflicts, required slots, and module compatibility; file
|
||||
materialization still occurs afterward.
|
||||
|
||||
For `run --json`, the CLI constructs and encodes its private run-result receipt
|
||||
after a successful runner result is available, before it publishes logical
|
||||
output files. It writes the prepared receipt to standard output only after
|
||||
@@ -166,8 +176,9 @@ is discoverable.
|
||||
- **internal/cli/production_contract_test.go** covers registrar composition,
|
||||
production catalog contents, assets, and representative configuration
|
||||
validation.
|
||||
- **internal/cli/reference_contract_test.go** covers CLI reference overrides,
|
||||
origin separation, and materialization boundaries.
|
||||
- **internal/cli/reference_contract_test.go** covers scoped CLI reference
|
||||
expansion, specificity, bind/unbind conflicts, generated-reference
|
||||
protection, origin separation, and materialization boundaries.
|
||||
- **internal/cli/state_hardening_test.go** covers safe run identity, state
|
||||
roots, and failure ordering.
|
||||
|
||||
|
||||
@@ -29,13 +29,17 @@ The D&D registrar registers the family’s artifact codecs, extractors, typed
|
||||
append-order mergers, normalizers, validators, prompt assets, fallback LLM
|
||||
profile asset, and default validator chains. Each extractor and normalizer has
|
||||
a stable module spec, explicit execution class, strict option decoding, and a
|
||||
typed builder. Scene chunking, every extractor, and NPC, location, and item-registry
|
||||
normalization are registered as `llm_backed`; the remaining current D&D mergers
|
||||
and normalizers are `deterministic`. The metadata is available to catalog inspection and
|
||||
resolved-pipeline debug data and determines which selected bindings inherit the
|
||||
pipeline profile. The registry normalizers use `single_response_v1`, forwarding
|
||||
corrections to their reconciliation completion and retaining the accepted raw
|
||||
proposal only as an owned model candidate. Configuration remains the canonical owner of the exact keys,
|
||||
typed builder. Scene chunking, every extractor, and NPC, location, and
|
||||
item-registry normalization are registered as `llm_backed`; the remaining
|
||||
current D&D mergers and normalizers are `deterministic`. The metadata is
|
||||
available to catalog inspection and resolved-pipeline debug data and determines
|
||||
which selected bindings inherit the pipeline profile. The registry normalizers
|
||||
use `single_response_v1`, forwarding corrections to their reconciliation
|
||||
completion and retaining the accepted raw proposal only as an owned model
|
||||
candidate. When deterministic proposal safety or typed application rejects a
|
||||
group, they provide separate model-facing prose for a corrective module retry;
|
||||
internal issue categories, reason codes, and operator messages remain
|
||||
diagnostic-only. Configuration remains the canonical owner of the exact keys,
|
||||
profile precedence, and validator order.
|
||||
|
||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||
@@ -117,14 +121,27 @@ presentation, and final ephemeral generic transcript windows. These orders and
|
||||
cache controls are prompt behavior; change them only through the owning
|
||||
manifest and prompt declaration.
|
||||
|
||||
NPC, item, and location registry reconciliation translate shared proposal
|
||||
safety categories into bounded prose that references only the response-local
|
||||
duplicate-group ordinals and candidate handles. Item reconciliation appends its
|
||||
typed rule that currency may be consolidated only with aliases of the same
|
||||
denomination and never with non-currency items. The next normalize attempt
|
||||
receives that prose with the exact defective proposal under the shared
|
||||
replacement-request protocol. Structurally invalid output has no valid proposal
|
||||
candidate and receives a fresh feedback-free attempt instead. If the stage
|
||||
budget is exhausted, safe groups stay applied, unsafe groups stay separate, and
|
||||
one fallback warning summarizes the final defect without raw model content.
|
||||
|
||||
## Evidence, Candidates, And Normalization
|
||||
|
||||
The current transcript is the only durable evidence source. Extractors assign
|
||||
the current source identity, preserve candidate evidence ranges for validators,
|
||||
and canonically order or remove exact duplicate ranges without asking the
|
||||
model to repair semantic errors. Campaign context and generated artifacts may
|
||||
ground names or control routing, but they never establish evidence for a D&D
|
||||
result.
|
||||
the current source identity and losslessly order any reversed range whose two
|
||||
endpoints resolve in that source, using transcript position rather than numeric
|
||||
unit-ID order. They then canonically order ranges and remove exact duplicates.
|
||||
This routine canonicalization does not request a retry or emit a warning.
|
||||
Unresolvable or wrong-source ranges remain unchanged for validators to reject.
|
||||
Campaign context and generated artifacts may ground names or control routing,
|
||||
but they never establish evidence for a D&D result.
|
||||
|
||||
Default chains keep responsibilities separate: structural validators assess the
|
||||
candidate, source-reference validators resolve cited ranges against the current
|
||||
@@ -134,18 +151,115 @@ relatedness validators report advisory evidence concerns. The configured order
|
||||
is documented in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
|
||||
Every D&D rejection describes the correction in transcript-grounded domain
|
||||
terms, using contextual names, artifact fields, and source segment ranges when
|
||||
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
||||
hashes, validator module keys, or reason codes. Those identifiers remain in
|
||||
ordinary validation provenance; only the actionable semantic guidance is
|
||||
eligible for the correction prompt.
|
||||
The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the
|
||||
D&D family's LLM-backed review of only combat versus non-combat classification.
|
||||
It selects the shared combat-policy prompt fragment and asks the model to
|
||||
classify the current chunk independently as `combat` or `non_combat` without
|
||||
receiving the proposed scene kind. Deterministic code compares that
|
||||
classification with the proposed kind and either approves it or produces the
|
||||
appropriate correction guidance. This keeps every schema-valid classification
|
||||
interpretable and avoids anchoring the reviewer on the producer's answer. It
|
||||
does not assess titles, summaries, non-combat subtype, or scene boundaries;
|
||||
deferred boundary-coherence review remains separate. It is opt-in;
|
||||
[Configuration](../config.md) owns selection and retry/failure behavior.
|
||||
|
||||
### Combat-semantics provider evaluation
|
||||
|
||||
The human-reviewed corpus at
|
||||
`internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json`
|
||||
owns the proposed kind, expected combat classification, and reviewer rationale
|
||||
for each synthetic case. Its package test validates the fixture contract only.
|
||||
Provider evaluation remains an explicit maintainer operation and must not be
|
||||
added to the default offline test suite.
|
||||
|
||||
Use the following protocol before proposing default-chain inclusion:
|
||||
|
||||
1. Record the Notarius commit, prompt and schema fingerprints, provider, model,
|
||||
profile settings, reasoning effort, structural-repair setting, number of
|
||||
repetitions, and evaluation date before collecting results. Do not revise
|
||||
expected classifications merely to agree with provider output; a substantive
|
||||
corpus correction requires independent human review.
|
||||
2. Exercise the production validator construction and prompt assets from an
|
||||
explicitly invoked, disposable evaluation driver or test in the validator
|
||||
package. For each corpus case, construct transcript source units from the
|
||||
listed IDs and text, assign matching per-unit source references, and use
|
||||
`source.MaterializeChunkPlan` with one range spanning those units. Construct
|
||||
exactly one scene whose ID and source range match that chunk and whose kind
|
||||
is the case's `proposed_kind`; title and summary may use fixed placeholders
|
||||
because the validator neither receives nor evaluates them. Invoke the typed
|
||||
validator at extract stage through the production registry and scheduled LLM
|
||||
client. Do not commit provider credentials, generated source material, or an
|
||||
always-on live test.
|
||||
3. Compare the model classification with `expected_classification`, then verify
|
||||
that its deterministic comparison with `proposed_kind` yields the expected
|
||||
approval or rejection direction.
|
||||
Record an unexpected approval of an expected rejection as a false
|
||||
acceptance, an unexpected rejection of an expected approval as a false
|
||||
rejection, and any validator execution failure separately from semantic
|
||||
accuracy. Retain per-case results so repeated trials and systematic failure
|
||||
modes remain visible.
|
||||
4. Evaluate producer correction separately with representative complete
|
||||
scene-description runs configured as shown in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
For every initial semantic rejection, record whether the next producer
|
||||
attempt returns the requested combat status and is approved. Do not count a
|
||||
PromptKit structural repair as a producer-correction attempt.
|
||||
5. Run the correction evaluation with debug capture enabled and without reused
|
||||
extraction checkpoints. Record added validator and producer calls, elapsed
|
||||
latency, and cumulative prompt, completion, cached, and total token usage
|
||||
from the debug attempt records. Compare these values with an otherwise
|
||||
identical run whose scene-description chain omits the semantic validator.
|
||||
|
||||
The default-chain review must consider classification error, false acceptance,
|
||||
false rejection, execution failure, producer-correction success, added calls,
|
||||
latency, and token use together. A structurally successful provider run alone
|
||||
is not evidence that the validator should become a default.
|
||||
|
||||
Every producer-correctable D&D rejection describes all currently detectable
|
||||
corrections in transcript-grounded domain terms, using contextual names,
|
||||
model-owned artifact fields, and source segment ranges when useful. Validators
|
||||
collect independent record defects in one pass so one retry does not merely
|
||||
reveal the next issue. Shared D&D diagnostic helpers keep repeated rules and
|
||||
record descriptions stable, de-duplicated, and bounded; each artifact family
|
||||
continues to own the semantic rule and its prose.
|
||||
|
||||
Operator diagnostics and model guidance are separate products of the same
|
||||
assessment. Operator messages may use typed paths, reason details, and opaque
|
||||
application identities. Correction guidance must not copy those messages or
|
||||
ask the model to reproduce durable entity IDs, hashes, validator module keys,
|
||||
reason codes, or Go field paths. A registry-normalization rejection instead
|
||||
speaks in terms of the duplicate-group proposal response the normalizer can
|
||||
actually revise. Normalization-only deterministic invariants retain useful
|
||||
operator detail but do not imply that a model controls derived ordering or
|
||||
identity. Only bounded actionable semantic guidance is eligible for a
|
||||
correction prompt.
|
||||
|
||||
Private LLM schemas use simple enums for closed categorical fields when the
|
||||
provider-compatible shape can express the rule directly. Deterministic typed
|
||||
validators retain the same checks as defense in depth and for non-LLM
|
||||
producers. Private schemas keep every property required and avoid optional
|
||||
properties, `uniqueItems`, and conditional cross-field logic.
|
||||
|
||||
Item-occurrence shape validation groups repeated holder mistakes by occurrence
|
||||
kind and gives the producer the required JSON null/non-null relationship. It
|
||||
identifies affected records by contextual item name and cited transcript range,
|
||||
never by the deterministically attached durable item ID. Holder mistakes remain
|
||||
semantic rejections rather than silent rewrites because changing a holder can
|
||||
also change the meaning of the occurrence kind.
|
||||
|
||||
Enemy-event extraction additionally rejects a second `engaged` observation for
|
||||
the same comparison identity within one scene-scoped result. Normalization may
|
||||
combine results from distinct scenes, so it intentionally does not apply that
|
||||
rule. Configuration owns the exact validator key and chain position.
|
||||
|
||||
Extraction source-reference validators share one full-span chunk-containment
|
||||
policy. After ordinary reference validity succeeds, the policy resolves both
|
||||
endpoints through document order and requires every source unit in the
|
||||
inclusive range to be present in the current chunk. It does not assume numeric
|
||||
unit-ID ordering, mutate input, or weaken wrong-source and unresolved-reference
|
||||
validation. Scene descriptions remain separate because their validator owns an
|
||||
exact one-scene range contract rather than general extraction containment.
|
||||
|
||||
Normalizers are deterministic for spells, combat turns, item occurrences, NPC
|
||||
occurrences, scene descriptions, enemy events, and location occurrences. They
|
||||
canonicalize display values and evidence, use source-document order for stable
|
||||
@@ -158,13 +272,16 @@ proposal to reconcile identity groups.
|
||||
|
||||
The three registry normalizers instantiate the domain-neutral
|
||||
`internal/framework/semanticreconcile` engine with default bounds. Each
|
||||
eligible candidate receives a contiguous, one-based `candidate_id` for that
|
||||
request. The model sees that handle, the candidate label and source-free
|
||||
evidence ranges, plus bounded transcript windows; it returns only duplicate
|
||||
eligible candidate receives a contiguous, one-based `candidate_number` for that
|
||||
request, and candidate material declares the exact inclusive range. The model
|
||||
sees that handle, the candidate label and source-free evidence ranges, plus
|
||||
bounded transcript windows; it returns only duplicate
|
||||
groups of supplied handles and one supplied canonical handle per group. It
|
||||
never returns names, evidence, durable IDs, or replacement records. Identical
|
||||
labels and evidence remain independently selectable because their handles are
|
||||
distinct.
|
||||
distinct. Transcript unit `id` values and evidence `start_unit_id` and
|
||||
`end_unit_id` values are source positions in a separate namespace and are
|
||||
never valid candidate numbers.
|
||||
|
||||
The generic core owns the mandatory handle protocol, candidate and transcript
|
||||
presentation, the private response schema, source-reference validation,
|
||||
|
||||
@@ -247,8 +247,12 @@ redaction boundary.
|
||||
Structural repair does not replace pipeline retry behavior: a binding's
|
||||
configured retry count reruns its complete stage attempt after an operational
|
||||
or structural error, module-requested retry, or actionable semantic rejection.
|
||||
The pipeline owns attempt lifecycle, validation chains, and retry diagnostics;
|
||||
see [Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||
An actionable module-requested retry and a validator rejection both use the
|
||||
same correction payload when the producer exposes an exact latest response;
|
||||
feedback-free module retries reconstruct the ordinary request without appended
|
||||
messages. The pipeline owns attempt lifecycle, validation chains, and retry
|
||||
diagnostics; see
|
||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||
[binding reference](../config.md#module-bindings-and-validators).
|
||||
|
||||
## Timeout Ownership
|
||||
|
||||
@@ -95,19 +95,24 @@ source-backed artifact-family normalizer projects its deterministic records
|
||||
into contextual candidates and owned typed record envelopes, supplies its
|
||||
chosen prompt identity and resolved LLM profile, and constructs an engine with
|
||||
explicit limits. The core filters invalid evidence, assigns contiguous
|
||||
request-local integer handles, renders bounded candidate and transcript
|
||||
materials, invokes the structured-completion boundary, and assesses the
|
||||
returned duplicate groups into a stable non-overlapping plan.
|
||||
one-based request-local candidate numbers, renders bounded candidate and
|
||||
transcript materials, invokes the structured-completion boundary, and assesses
|
||||
the returned duplicate groups into a stable non-overlapping plan. Candidate
|
||||
material declares the exact inclusive number range for the request. Transcript
|
||||
unit IDs and evidence range endpoints remain source positions in a separate
|
||||
namespace and are never valid candidate numbers.
|
||||
|
||||
The normalizer then applies that plan through a typed `ApplicationPolicy`. The
|
||||
core preserves ungrouped records, contribution order, and provenance while the
|
||||
artifact family owns group guards, field and evidence consolidation, durable
|
||||
ID derivation, retry and fallback presentation, classified diagnostics, and postconditions.
|
||||
Request-local handles do not enter the typed value or durable artifact. Fewer
|
||||
than two eligible candidates skips model invocation; exceeding a candidate or
|
||||
combined-material bound preserves the deterministic result under the family's
|
||||
fallback policy. Provider, transport, cancellation, and context-construction
|
||||
failures remain execution errors.
|
||||
ID derivation, retry and fallback presentation, classified diagnostics, and
|
||||
postconditions.
|
||||
These candidate numbers are the concrete private representation of ADR-0013's
|
||||
request-local handles; they do not enter the typed value or durable artifact.
|
||||
Fewer than two eligible candidates skips model invocation; exceeding a
|
||||
candidate or combined-material bound preserves the deterministic result under
|
||||
the family's fallback policy. Provider, transport, cancellation, and
|
||||
context-construction failures remain execution errors.
|
||||
|
||||
When the engine actually makes a proposal call, its typed result carries the
|
||||
owned exact proposal response under the same correction contract as other
|
||||
@@ -115,6 +120,16 @@ eligible producers. Deterministic skip, limit, and fallback outcomes carry no
|
||||
model candidate, so a later rejection applies terminal policy without spending
|
||||
an ineffective semantic retry.
|
||||
|
||||
When proposal assessment or typed group application rejects a structurally
|
||||
valid group, the normalizer may return its safe partial value with a
|
||||
module-requested retry. A feedback-capable directive supplies bounded
|
||||
model-facing correction guidance separately from operator diagnostics and
|
||||
retains the exact proposal response as its candidate. The shared stage retry
|
||||
mechanism appends that response and guidance to a fresh complete request. A
|
||||
structurally invalid completion has no valid candidate and therefore requests a
|
||||
feedback-free fresh attempt. On exhaustion, only the final safe fallback and
|
||||
its bounded process diagnostic advance to validation.
|
||||
|
||||
The core supplies a conservative generic prompt and the single private
|
||||
response schema. A domain prompt may substitute its semantic instructions but
|
||||
mounts the core-owned protocol and candidate/transcript presentation assets.
|
||||
|
||||
@@ -97,6 +97,13 @@ codec, checks its complete schema and media identity, and records a content
|
||||
digest plus bounded producer provenance. A missing, ambiguous, invalid, or
|
||||
incompatible producer prevents the consumer step from starting.
|
||||
|
||||
Resolution receives only exact stage-and-lane operational reference overrides.
|
||||
The CLI may offer broader pipeline- or lane-scoped selectors, but expands and
|
||||
arbitrates those before entering the framework. External overrides are applied
|
||||
after configured external defaults and local bindings. They cannot coexist
|
||||
with a generated binding for the same target and slot, and external unbinds do
|
||||
not remove generated handoffs.
|
||||
|
||||
## Execution And Ordering
|
||||
|
||||
The runner validates its input, installs no-op state collaborators when none
|
||||
@@ -154,18 +161,25 @@ complete validation chain. It preserves terminal diagnostics only from the final
|
||||
or rejected attempt, plus one fixed validation-incomplete warning per validator whose execution
|
||||
budget was exhausted under `warn_continue`. Cancellation stops retries.
|
||||
Normalizer-specific retry directives consume this same budget and validate any
|
||||
final safe fallback through the normalizer chain.
|
||||
final safe fallback through the normalizer chain. A directive may carry bounded
|
||||
correction guidance only when it also exposes the exact latest
|
||||
`single_response_v1` candidate. The state machine then uses the same replacement
|
||||
request shape as validator correction. A directive without guidance clears any
|
||||
prior correction and starts a fresh attempt, which preserves structural retry
|
||||
behavior when no valid response exists.
|
||||
|
||||
The artifact-neutral producer-attempt state machine owns that shared budget,
|
||||
attempt provenance, semantic-correction material, and terminal-policy
|
||||
selection. It accepts producer and complete-validation closures, so artifact
|
||||
materialization, cache handling, checkpoints, and debug output stay at the
|
||||
operation boundary. It distinguishes operational, structural, module-requested,
|
||||
and semantic retries. A semantic retry is available only for a valid latest
|
||||
`single_response_v1` candidate; a deterministic or no-model rejection instead
|
||||
settles the semantic policy immediately. Structural-output errors alone use the
|
||||
structural policy, and validation failure without rejection settles the
|
||||
validator-failure policy without regenerating the producer.
|
||||
and validator-semantic retries. Model feedback from either semantic source is
|
||||
available only for a valid latest `single_response_v1` candidate. A
|
||||
deterministic or no-model validator rejection instead settles the semantic
|
||||
policy immediately, while a feedback-free module directive remains an ordinary
|
||||
fresh retry. Structural-output errors alone use the structural policy, and
|
||||
validation failure without rejection settles the validator-failure policy
|
||||
without regenerating the producer.
|
||||
|
||||
Chunk planning uses this state machine for generated plans. A rejected or
|
||||
validation-incomplete automatic cache hit is not model material and therefore
|
||||
@@ -224,6 +238,8 @@ Debug recording is attempt-scoped and application-owned. A failure to persist
|
||||
required debug data is a framework error. State roots, persistence, reason-code
|
||||
meanings, resume, and cleanup are intentionally owned by
|
||||
[Run State Internals](state.md) and [Operations](../operations.md).
|
||||
Extract-validator trace scopes include the current chunk ordinal so concurrent
|
||||
chunks cannot overwrite one another's validator attempts or LLM artifacts.
|
||||
|
||||
## Invariants To Preserve
|
||||
|
||||
|
||||
@@ -201,6 +201,13 @@ they are not model instructions. The framework constructs model-facing retry
|
||||
text only from the semantic guidance and fails the contract rather than
|
||||
inventing or truncating missing guidance.
|
||||
|
||||
An LLM-backed module may also request a feedback-aware retry when its own
|
||||
deterministic translation or typed safety policy rejects a structurally valid
|
||||
model response. It must supply model-facing guidance separately from its
|
||||
reason code, operator message, and fallback diagnostics, together with the
|
||||
exact `single_response_v1` candidate. A feedback-free module retry remains
|
||||
valid when no exact candidate exists.
|
||||
|
||||
Default validator chains are production composition policy and are registered
|
||||
centrally by stage and module. Configuration may replace a stage-local default,
|
||||
including with an explicitly empty chain. Configured validator order is
|
||||
@@ -222,12 +229,14 @@ the two budgets must remain separate.
|
||||
|
||||
An LLM-backed producer can participate in semantic correction only when it
|
||||
declares `single_response_v1` and returns the exact one response that directly
|
||||
controlled its candidate. On an actionable rejection, the framework rebuilds
|
||||
the ordinary request and appends only the latest defective response as an
|
||||
`assistant` message plus one aggregated `user` correction message. This is a
|
||||
fresh replacement request, not a growing conversation. The retry budgets,
|
||||
terminal policy, and sensitive-data rationale are recorded in
|
||||
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md).
|
||||
controlled its candidate. On an actionable validator rejection or
|
||||
feedback-capable module retry, the framework rebuilds the ordinary request and
|
||||
appends only the latest defective response as an `assistant` message plus one
|
||||
aggregated `user` correction message. This is a fresh replacement request, not
|
||||
a growing conversation. The retry budgets, terminal policy, and sensitive-data
|
||||
rationale are recorded in
|
||||
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md) and
|
||||
[ADR-0016](../adr/0016-feedback-aware-module-requested-retries.md).
|
||||
|
||||
When a model selects an application entity, callers must supply a contextual
|
||||
selection and deterministically attach the opaque application identity whenever
|
||||
|
||||
84
docs/releases/v0.6.0.md
Normal file
84
docs/releases/v0.6.0.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Notarius v0.6.0
|
||||
|
||||
This release improves the reliability and ergonomics of unattended,
|
||||
subprocess-driven D&D extraction pipelines.
|
||||
|
||||
## Summary
|
||||
|
||||
Notarius now gives models more precise, semantically useful correction guidance
|
||||
when deterministic or model-backed validation rejects an otherwise structured
|
||||
candidate. Semantic reconciliation retries identify candidates with compact,
|
||||
request-local numbers, preserve valid candidates when a proposal cannot be
|
||||
repaired, and report bounded process warnings when reconciliation is
|
||||
incomplete. D&D extraction also canonicalizes safely resolvable reversed source
|
||||
ranges and applies more consistent schema, prompt, and validator constraints,
|
||||
improving successful operation with smaller models without weakening evidence
|
||||
validation.
|
||||
|
||||
An optional LLM-backed combat-semantics validator can review whether scene
|
||||
descriptions correctly consolidate combat and apply the `combat` kind. For
|
||||
subprocess callers, command-line references now follow the pipeline-scoped
|
||||
configuration model: a shared reference can be supplied once and automatically
|
||||
reach every compatible selected target, while lane- and binding-specific forms
|
||||
remain available for exceptions.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- The meaning of an unqualified `--reference slot=path` or
|
||||
`--without-reference slot` selector has changed. It now applies to every
|
||||
selected pipeline target that declares the slot instead of requiring exactly
|
||||
one matching target. Review callers that relied on ambiguity rejection or a
|
||||
unique implicit target; the current selector contract is documented in the
|
||||
[CLI reference](../cli.md).
|
||||
- The stage-wide `merge.slot` CLI reference shorthand has been removed. Use
|
||||
`lane.merge.slot` for an exact merge binding, `lane.slot` for all compatible
|
||||
bindings in one lane, or an unqualified `slot` for pipeline scope.
|
||||
- Command-line file references cannot replace, remove, or coexist with a
|
||||
generated artifact handoff for the same concrete target and slot. Resolution
|
||||
reports the conflict so the caller can narrow or remove the selector.
|
||||
- The private LLM-facing semantic-reconciliation contract now uses contiguous
|
||||
`candidate_number` values rather than application identities. This is not a
|
||||
durable or operator-configurable contract and requires no operator action.
|
||||
- Durable D&D artifact schemas, the `notarius.run-result.v2` receipt,
|
||||
`notarius.warnings.v2`, and `notarius.diagnostics.v1` are unchanged from
|
||||
`v0.5.0`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
1. Update subprocess commands to provide shared inputs once with
|
||||
`--reference slot=path`, and convert any stage-wide `merge.slot` selector to
|
||||
an explicit supported scope. The [subprocess consumer guide](../consumers/subprocess.md)
|
||||
and [complete D&D consumer guide](../consumers/dnd-pipeline.md) show the
|
||||
current invocation pattern.
|
||||
2. Review CLI reference overrides that overlap generated same-run references;
|
||||
narrow or remove an external selector rather than attempting to replace the
|
||||
generated handoff.
|
||||
3. Run `notarius config validate --config <path> --pipeline <id>` for every
|
||||
deployed pipeline configuration.
|
||||
4. Optionally add
|
||||
`extract/dnd/scene-descriptions/combat_semantics` to the scene-description
|
||||
validator chain where the deployment wants LLM-backed combat-scene review;
|
||||
see the [configuration reference](../config.md).
|
||||
5. Perform a representative run with the deployed model profile and inspect
|
||||
the machine-readable receipt, warnings, and diagnostics before promotion.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added feedback-aware, module-requested retry support for normalizers while
|
||||
keeping retry budgets bounded and preserving attempt diagnostics.
|
||||
- Reworked semantic reconciliation around compact, contiguous candidate
|
||||
numbers, explicit namespace guidance, complete proposal validation, and
|
||||
model-facing retry feedback that omits opaque internal identities.
|
||||
- Preserved candidates separately when an exhausted semantic proposal cannot
|
||||
be safely applied, with a bounded process warning describing the fallback.
|
||||
- Canonicalized safely resolvable reversed D&D source-reference endpoints
|
||||
before deterministic coverage validation and clarified the shared evidence
|
||||
prompt.
|
||||
- Improved D&D schemas, prompts, and validators with closed-value constraints,
|
||||
contextual correction guidance, shared source-range coverage logic, and
|
||||
clearer item-holder transition rules.
|
||||
- Added the optional D&D scene-description combat-semantics validator, shared
|
||||
combat policy assets, evaluation fixtures, and retry-path coverage.
|
||||
- Made unqualified CLI references pipeline-scoped, added hierarchical lane and
|
||||
exact-binding selectors, defined deterministic override precedence, and
|
||||
documented first-class subprocess use.
|
||||
@@ -14,47 +14,9 @@ eligibility, and the separation of actionable process warnings from quality
|
||||
diagnostics. The remaining near-term work applies those completed foundations
|
||||
to domain review and empirical evaluation.
|
||||
|
||||
### D&D Combat Scene Semantic Validation
|
||||
|
||||
- Add an optional production LLM-backed D&D validator that determines whether
|
||||
proposed scene boundaries and classifications represent substantive active
|
||||
combat correctly. Its central quality goal is that active combat is kept in
|
||||
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||
- Resolve the validator's exact target before implementation. The current
|
||||
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||
`recap`, and `meta` classification. The preferred initial placement is
|
||||
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||
can compare one proposed kind with the corresponding transcript chunk.
|
||||
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||
question that can be answered from the complete transcript and proposed
|
||||
range map, such as whether one continuous combat was fragmented across
|
||||
inappropriate scene boundaries. Do not duplicate the same classification
|
||||
judgment at both stages. Moving classification into chunk-plan annotations
|
||||
would change the deliberately minimal, annotation-free chunk contract and
|
||||
requires an explicit architecture review before it is selected.
|
||||
- Validate both false negatives and false positives: a non-combat kind must not
|
||||
omit substantive active combat, and a combat kind must be supported by such
|
||||
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||
extraction runs only for an exact `combat` scene classification; semantic
|
||||
review improves the upstream classification but does not replace that gate.
|
||||
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||
structured response schema, and let PromptKit repair structural validator
|
||||
output within its bounded budget. A contract-invalid final validator response
|
||||
is a validator execution failure, not a semantic rejection and not a reason
|
||||
to recursively validate the validator.
|
||||
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||
added calls, latency, and token cost before placing it in the production
|
||||
default chain.
|
||||
- An ADR is not required if classification remains owned by
|
||||
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||
Create or supersede an ADR if the work transfers scene classification into
|
||||
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||
contract.
|
||||
Near-term reliability work should now be selected from the concrete evaluation
|
||||
and extension opportunities below. The retry, validation, and subprocess
|
||||
foundations described above are implemented current behavior.
|
||||
|
||||
## Near-Term D&D Pipeline
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -14,20 +15,16 @@ import (
|
||||
|
||||
func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
selector string
|
||||
only []string
|
||||
wantStage pipeline.ModuleStage
|
||||
wantLane string
|
||||
wantSlot string
|
||||
name string
|
||||
selector string
|
||||
want []string
|
||||
}{
|
||||
{name: "flat", selector: "alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "chunk", selector: "chunk.chunk-slot", wantStage: pipeline.StageChunk, wantSlot: "chunk-slot"},
|
||||
{name: "merge", selector: "merge.alpha-merge", only: []string{"alpha"}, wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane", selector: "alpha.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane extract", selector: "alpha.extract.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane merge", selector: "alpha.merge.alpha-merge", wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", wantStage: pipeline.StageNormalize, wantLane: "alpha", wantSlot: "alpha-normalize"},
|
||||
{name: "pipeline", selector: "shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared", "beta.extract.shared", "beta.merge.shared", "beta.normalize.shared"}},
|
||||
{name: "chunk", selector: "chunk.chunk-slot", want: []string{"chunk.chunk-slot"}},
|
||||
{name: "lane", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
|
||||
{name: "lane extract", selector: "alpha.extract.alpha-slot", want: []string{"alpha.extract.alpha-slot"}},
|
||||
{name: "lane merge", selector: "alpha.merge.alpha-merge", want: []string{"alpha.merge.alpha-merge"}},
|
||||
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", want: []string{"alpha.normalize.alpha-normalize"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -37,70 +34,135 @@ func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve selector: %v", err)
|
||||
}
|
||||
if len(overrides) != 1 {
|
||||
t.Fatalf("overrides = %#v, want one binding", overrides)
|
||||
if got := referenceContractBindingLabels(overrides); !slices.Equal(got, tt.want) {
|
||||
t.Fatalf("binding targets = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
got := overrides[0]
|
||||
if got.Stage != tt.wantStage || got.LaneID != tt.wantLane || got.SlotName != tt.wantSlot || got.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding = %#v, want %s/%s/%s from CLI", got, tt.wantStage, tt.wantLane, tt.wantSlot)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectAmbiguityWithSpecificSuggestions(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
want []string
|
||||
}{
|
||||
{name: "flat shared slot", selector: "shared", want: []string{"alpha.extract.shared", "beta.extract.shared"}},
|
||||
{name: "lane shared slot", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
|
||||
{name: "all mergers", selector: "merge.shared", want: []string{"alpha.merge.shared", "beta.merge.shared"}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("resolve selector succeeded, want ambiguity error")
|
||||
}
|
||||
for _, fragment := range tt.want {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %q, want suggestion %q", err, fragment)
|
||||
for _, binding := range overrides {
|
||||
if binding.Source != "reference.txt" || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding = %#v, want CLI source", binding)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRespectSelectedLanesBeforeMaterialization(t *testing.T) {
|
||||
func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
requests := []cliReferenceRequest{
|
||||
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-first.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "pipeline-final.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "lane.txt"},
|
||||
{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "binding.txt"},
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
want := map[string]string{
|
||||
"alpha.extract.shared": "binding.txt",
|
||||
"alpha.merge.shared": "lane.txt",
|
||||
"alpha.normalize.shared": "lane.txt",
|
||||
"beta.extract.shared": "pipeline-final.txt",
|
||||
"beta.merge.shared": "pipeline-final.txt",
|
||||
"beta.normalize.shared": "pipeline-final.txt",
|
||||
}
|
||||
for _, binding := range overrides {
|
||||
label := referenceContractBindingLabel(binding)
|
||||
if binding.Source != want[label] {
|
||||
t.Fatalf("binding %s source = %q, want %q", label, binding.Source, want[label])
|
||||
}
|
||||
delete(want, label)
|
||||
}
|
||||
if len(want) != 0 {
|
||||
t.Fatalf("missing bindings: %#v", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
|
||||
catalog := catalogFromRegistries(productionTestComponents(t).registries)
|
||||
sources := map[string]string{
|
||||
"party": "/references/party.txt",
|
||||
"players": "/references/players.txt",
|
||||
"glossary": "/references/glossary.txt",
|
||||
"spell_catalog": "/references/spells.json",
|
||||
}
|
||||
requests := make([]cliReferenceRequest, 0, len(sources))
|
||||
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
|
||||
requests = append(requests, cliReferenceRequest{
|
||||
Selector: mustParseReferenceSelector(t, slot, "--reference"),
|
||||
Source: sources[slot],
|
||||
})
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand complete D&D references: %v", err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
|
||||
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
|
||||
for _, binding := range overrides {
|
||||
actual[referenceContractBindingLabel(binding)] = binding
|
||||
}
|
||||
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matched := make(map[string]int, len(sources))
|
||||
for _, target := range targets {
|
||||
for slot, sourcePath := range sources {
|
||||
if _, ok := target.slots[slot]; !ok {
|
||||
continue
|
||||
}
|
||||
matched[slot]++
|
||||
label := targetLabel(target) + "." + slot
|
||||
binding, ok := actual[label]
|
||||
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
for slot := range sources {
|
||||
if matched[slot] < 2 {
|
||||
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
|
||||
}
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
|
||||
t.Fatalf("resolve complete D&D CLI references: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
only []string
|
||||
want string
|
||||
}{
|
||||
{name: "unselected lane", selector: "beta.extract.beta-slot", want: `reference lane "beta" is not selected`},
|
||||
{name: "pipeline slot", selector: "missing", want: `reference slot "missing" is not declared by any selected target`},
|
||||
{name: "lane slot", selector: "alpha.missing", want: `reference slot "missing" is not declared by selected lane "alpha"`},
|
||||
{name: "binding slot", selector: "alpha.extract.missing", want: `reference slot "missing" is not declared`},
|
||||
{name: "former merge shorthand", selector: "merge.shared", want: `reference lane "merge" is not selected`},
|
||||
{name: "unselected lane", selector: "beta.extract.beta-slot", only: []string{"alpha"}, want: `reference lane "beta" is not selected`},
|
||||
{name: "unknown lane", selector: "missing.extract.beta-slot", want: `reference lane "missing" is not selected`},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", []string{"alpha"}, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
|
||||
selector := mustParseReferenceSelector(t, tt.selector, "--reference")
|
||||
_, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || strings.Contains(err.Error(), "missing.txt") {
|
||||
t.Fatalf("error = %v, want selection failure before file access", err)
|
||||
t.Fatalf("error = %v, want selection failure containing %q before file access", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -131,40 +193,50 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceOverridesUseFinalExactTargetBinding(t *testing.T) {
|
||||
func TestReferenceBindAndUnbindSpecificity(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
alphaShared, err := parseReferenceSelector("alpha.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
betaShared, err := parseReferenceSelector("beta.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{
|
||||
{Selector: alphaShared, Source: "alpha-first.txt"},
|
||||
{Selector: alphaShared, Source: "alpha-final.txt"},
|
||||
{Selector: betaShared, Source: "beta-only.txt"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve pipeline: %v", err)
|
||||
}
|
||||
alpha := referenceContractLane(t, effective.ResolvedPipeline, "alpha")
|
||||
beta := referenceContractLane(t, effective.ResolvedPipeline, "beta")
|
||||
if source := referenceContractBindingSource(alpha.ExtractReferences.Bindings, "shared"); source != "alpha-final.txt" {
|
||||
t.Fatalf("alpha shared source = %q, want final exact-target override", source)
|
||||
}
|
||||
if source := referenceContractBindingSource(beta.ExtractReferences.Bindings, "shared"); source != "beta-only.txt" {
|
||||
t.Fatalf("beta shared source = %q, want target-specific override", source)
|
||||
}
|
||||
t.Run("specific unbind carves out broad binding", func(t *testing.T) {
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: "shared.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--without-reference")}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := referenceContractBindingLabels(overrides); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
|
||||
t.Fatalf("overrides = %#v, want all shared targets except alpha extract", got)
|
||||
}
|
||||
if got := referenceContractUnbindLabels(unbinds); !slices.Equal(got, []string{"alpha.extract.shared"}) {
|
||||
t.Fatalf("unbinds = %#v, want alpha extract", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("specific binding restores broad unbind", func(t *testing.T) {
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.extract.shared", "--reference"), Source: "alpha.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "shared", "--without-reference")}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := referenceContractBindingLabels(overrides); !slices.Equal(got, []string{"alpha.extract.shared"}) {
|
||||
t.Fatalf("overrides = %#v, want alpha extract", got)
|
||||
}
|
||||
if got := referenceContractUnbindLabels(unbinds); slices.Contains(got, "alpha.extract.shared") || len(got) != 5 {
|
||||
t.Fatalf("unbinds = %#v, want all shared targets except alpha extract", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("same specificity conflicts", func(t *testing.T) {
|
||||
_, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog,
|
||||
[]cliReferenceRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--reference"), Source: "alpha.txt"}},
|
||||
[]cliReferenceUnbindRequest{{Selector: mustParseReferenceSelector(t, "alpha.shared", "--without-reference")}},
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "same specificity") {
|
||||
t.Fatalf("error = %v, want same-specificity conflict", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReferenceUnbindsRemoveOptionalAndProtectRequiredSlots(t *testing.T) {
|
||||
@@ -257,6 +329,52 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
profile := cfg.Pipelines["demo"]
|
||||
alpha := profile.Artifacts["alpha"]
|
||||
beta := profile.Artifacts["beta"]
|
||||
alpha.Extract.References["shared"] = pipeline.GeneratedReference("produce", "beta")
|
||||
profile.Artifacts = nil
|
||||
profile.Steps = []pipeline.PipelineStepProfile{
|
||||
{ID: "produce", Artifacts: map[string]pipeline.ArtifactLaneProfile{"beta": beta}},
|
||||
{ID: "consume", Artifacts: map[string]pipeline.ArtifactLaneProfile{"alpha": alpha}},
|
||||
}
|
||||
cfg.Pipelines["demo"] = profile
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
|
||||
t.Run("binding conflicts before file access", func(t *testing.T) {
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{
|
||||
Selector: mustParseReferenceSelector(t, "shared", "--reference"),
|
||||
Source: filepath.Join(t.TempDir(), "never-read.json"),
|
||||
}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand CLI reference: %v", err)
|
||||
}
|
||||
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") {
|
||||
t.Fatalf("resolve error = %v, want generated/external conflict before file access", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unbind leaves generated source intact", func(t *testing.T) {
|
||||
_, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, nil, []cliReferenceUnbindRequest{{
|
||||
Selector: mustParseReferenceSelector(t, "shared", "--without-reference"),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("expand CLI unbind: %v", err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceUnbinds: unbinds})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve generated reference with CLI unbind: %v", err)
|
||||
}
|
||||
binding := referenceContractFindBinding(referenceContractLane(t, effective.ResolvedPipeline, "alpha").ExtractReferences.Bindings, "shared")
|
||||
if binding == nil || binding.Artifact == nil || binding.Artifact.Step != "produce" || binding.Artifact.Lane != "beta" {
|
||||
t.Fatalf("generated binding = %#v, want preserved produce/beta handoff", binding)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReferenceTargetLookupUsesArtifactVariantsAndReportsMissingContext(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
full := referenceContractCatalog(t, true, true)
|
||||
@@ -368,7 +486,7 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
|
||||
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
if includeBetaMerger {
|
||||
@@ -444,6 +562,42 @@ func referenceContractBindingSource(bindings []pipeline.ReferenceBinding, slot s
|
||||
return ""
|
||||
}
|
||||
|
||||
func mustParseReferenceSelector(t *testing.T, value, flagName string) cliReferenceSelector {
|
||||
t.Helper()
|
||||
selector, err := parseReferenceSelector(value, flagName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return selector
|
||||
}
|
||||
|
||||
func referenceContractBindingLabels(bindings []pipeline.ReferenceBinding) []string {
|
||||
labels := make([]string, 0, len(bindings))
|
||||
for _, binding := range bindings {
|
||||
labels = append(labels, referenceContractBindingLabel(binding))
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func referenceContractBindingLabel(binding pipeline.ReferenceBinding) string {
|
||||
if binding.Stage == pipeline.StageChunk {
|
||||
return "chunk." + binding.SlotName
|
||||
}
|
||||
return binding.LaneID + "." + string(binding.Stage) + "." + binding.SlotName
|
||||
}
|
||||
|
||||
func referenceContractUnbindLabels(unbinds []pipeline.ReferenceUnbind) []string {
|
||||
labels := make([]string, 0, len(unbinds))
|
||||
for _, unbind := range unbinds {
|
||||
labels = append(labels, referenceContractBindingLabel(pipeline.ReferenceBinding{
|
||||
Stage: unbind.Stage,
|
||||
LaneID: unbind.LaneID,
|
||||
SlotName: unbind.SlotName,
|
||||
}))
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func referenceContractFindBinding(bindings []pipeline.ReferenceBinding, slot string) *pipeline.ReferenceBinding {
|
||||
for i := range bindings {
|
||||
if bindings[i].SlotName == slot {
|
||||
|
||||
@@ -177,7 +177,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
|
||||
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
||||
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
|
||||
if err := validateRunFlagValues(args); err != nil {
|
||||
@@ -1287,11 +1287,21 @@ type cliReferenceUnbindRequest struct {
|
||||
}
|
||||
|
||||
type cliReferenceSelector struct {
|
||||
Scope cliReferenceSelectorScope
|
||||
LaneID string
|
||||
Stage pipeline.ModuleStage
|
||||
SlotName string
|
||||
}
|
||||
|
||||
type cliReferenceSelectorScope uint8
|
||||
|
||||
const (
|
||||
cliReferenceScopePipeline cliReferenceSelectorScope = iota
|
||||
cliReferenceScopeLane
|
||||
cliReferenceScopeChunk
|
||||
cliReferenceScopeBinding
|
||||
)
|
||||
|
||||
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
@@ -1300,7 +1310,7 @@ func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
||||
for _, raw := range values {
|
||||
name, source, ok := strings.Cut(raw, "=")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
|
||||
return nil, fmt.Errorf("--reference must use slot=path, lane.slot=path, or lane.stage.slot=path")
|
||||
}
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
||||
@@ -1350,17 +1360,14 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
|
||||
}
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopePipeline, SlotName: strings.TrimSpace(parts[0])}, nil
|
||||
case 2:
|
||||
first := strings.TrimSpace(parts[0])
|
||||
slotName := strings.TrimSpace(parts[1])
|
||||
if first == string(pipeline.StageChunk) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeChunk, Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||
}
|
||||
if first == string(pipeline.StageMerge) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
|
||||
}
|
||||
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeLane, LaneID: first, SlotName: slotName}, nil
|
||||
case 3:
|
||||
laneID := strings.TrimSpace(parts[0])
|
||||
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
|
||||
@@ -1368,9 +1375,9 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
|
||||
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||
return cliReferenceSelector{Scope: cliReferenceScopeBinding, LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||
default:
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1391,37 +1398,153 @@ func resolveCLIReferenceRequests(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
||||
// Broad CLI selectors are only presentation syntax. Collapse them into one
|
||||
// highest-specificity action per concrete framework target before pipeline
|
||||
// resolution so the generic reference contract stays stage-and-lane exact.
|
||||
actions := make(map[cliReferenceTargetKey]resolvedCLIReferenceAction)
|
||||
for _, request := range referenceRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
matches, err := resolveCLIReferenceTargets(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
Source: request.Source,
|
||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||
})
|
||||
for _, target := range matches {
|
||||
candidate := resolvedCLIReferenceAction{
|
||||
kind: cliReferenceActionBind,
|
||||
selector: request.Selector,
|
||||
target: target,
|
||||
slotName: request.Selector.SlotName,
|
||||
source: request.Source,
|
||||
specificity: request.Selector.specificity(),
|
||||
}
|
||||
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
||||
for _, request := range unbindRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
matches, err := resolveCLIReferenceTargets(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
})
|
||||
for _, target := range matches {
|
||||
candidate := resolvedCLIReferenceAction{
|
||||
kind: cliReferenceActionUnbind,
|
||||
selector: request.Selector,
|
||||
target: target,
|
||||
slotName: request.Selector.SlotName,
|
||||
specificity: request.Selector.specificity(),
|
||||
}
|
||||
if err := mergeCLIReferenceAction(actions, candidate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolved := make([]resolvedCLIReferenceAction, 0, len(actions))
|
||||
for _, action := range actions {
|
||||
resolved = append(resolved, action)
|
||||
}
|
||||
sort.Slice(resolved, func(i, j int) bool {
|
||||
left, right := resolved[i], resolved[j]
|
||||
if left.target.laneID != right.target.laneID {
|
||||
return left.target.laneID < right.target.laneID
|
||||
}
|
||||
if left.target.stage != right.target.stage {
|
||||
return referenceStageOrder(left.target.stage) < referenceStageOrder(right.target.stage)
|
||||
}
|
||||
return left.slotName < right.slotName
|
||||
})
|
||||
|
||||
overrides := make([]pipeline.ReferenceBinding, 0, len(resolved))
|
||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(resolved))
|
||||
for _, action := range resolved {
|
||||
switch action.kind {
|
||||
case cliReferenceActionBind:
|
||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||
Stage: action.target.stage,
|
||||
LaneID: action.target.laneID,
|
||||
SlotName: action.slotName,
|
||||
Source: action.source,
|
||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||
})
|
||||
case cliReferenceActionUnbind:
|
||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||
Stage: action.target.stage,
|
||||
LaneID: action.target.laneID,
|
||||
SlotName: action.slotName,
|
||||
})
|
||||
}
|
||||
}
|
||||
return overrides, unbinds, nil
|
||||
}
|
||||
|
||||
type cliReferenceActionKind uint8
|
||||
|
||||
const (
|
||||
cliReferenceActionBind cliReferenceActionKind = iota
|
||||
cliReferenceActionUnbind
|
||||
)
|
||||
|
||||
type cliReferenceTargetKey struct {
|
||||
stage pipeline.ModuleStage
|
||||
laneID string
|
||||
slotName string
|
||||
}
|
||||
|
||||
type resolvedCLIReferenceAction struct {
|
||||
kind cliReferenceActionKind
|
||||
selector cliReferenceSelector
|
||||
target selectedReferenceTarget
|
||||
slotName string
|
||||
source string
|
||||
specificity int
|
||||
}
|
||||
|
||||
func mergeCLIReferenceAction(actions map[cliReferenceTargetKey]resolvedCLIReferenceAction, candidate resolvedCLIReferenceAction) error {
|
||||
key := cliReferenceTargetKey{stage: candidate.target.stage, laneID: candidate.target.laneID, slotName: candidate.slotName}
|
||||
current, ok := actions[key]
|
||||
if !ok || candidate.specificity > current.specificity {
|
||||
actions[key] = candidate
|
||||
return nil
|
||||
}
|
||||
if candidate.specificity < current.specificity {
|
||||
return nil
|
||||
}
|
||||
if candidate.kind != current.kind {
|
||||
return fmt.Errorf("reference target %q slot %q is both bound by %q and unbound by %q at the same specificity", targetLabel(candidate.target), candidate.slotName, current.selector.String(), candidate.selector.String())
|
||||
}
|
||||
actions[key] = candidate
|
||||
return nil
|
||||
}
|
||||
|
||||
func (selector cliReferenceSelector) specificity() int {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
return 0
|
||||
case cliReferenceScopeLane:
|
||||
return 1
|
||||
case cliReferenceScopeChunk, cliReferenceScopeBinding:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func (selector cliReferenceSelector) String() string {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
return selector.SlotName
|
||||
case cliReferenceScopeLane:
|
||||
return selector.LaneID + "." + selector.SlotName
|
||||
case cliReferenceScopeChunk:
|
||||
return "chunk." + selector.SlotName
|
||||
case cliReferenceScopeBinding:
|
||||
return selector.LaneID + "." + string(selector.Stage) + "." + selector.SlotName
|
||||
default:
|
||||
return selector.SlotName
|
||||
}
|
||||
}
|
||||
|
||||
type selectedReferenceTarget struct {
|
||||
laneID string
|
||||
stage pipeline.ModuleStage
|
||||
@@ -1638,114 +1761,77 @@ func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
|
||||
return slotSet
|
||||
}
|
||||
|
||||
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
|
||||
func resolveCLIReferenceTargets(targets []selectedReferenceTarget, selector cliReferenceSelector) ([]selectedReferenceTarget, error) {
|
||||
slotName := strings.TrimSpace(selector.SlotName)
|
||||
if slotName == "" {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
|
||||
return nil, fmt.Errorf("reference slot must not be empty")
|
||||
}
|
||||
if selector.Stage == pipeline.StageChunk {
|
||||
switch selector.Scope {
|
||||
case cliReferenceScopePipeline:
|
||||
matches := make([]selectedReferenceTarget, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by any selected target", slotName)
|
||||
}
|
||||
return matches, nil
|
||||
case cliReferenceScopeChunk:
|
||||
for _, target := range targets {
|
||||
if target.stage != pipeline.StageChunk {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
|
||||
}
|
||||
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
|
||||
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
|
||||
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
|
||||
return []selectedReferenceTarget{target}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reference chunk target is not selected")
|
||||
case cliReferenceScopeLane:
|
||||
laneSelected := false
|
||||
matches := make([]selectedReferenceTarget, 0, 3)
|
||||
for _, target := range targets {
|
||||
if target.laneID == selector.LaneID && target.stage == selector.Stage {
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||
}
|
||||
return target, nil
|
||||
if target.laneID != selector.LaneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
if strings.TrimSpace(selector.LaneID) != "" {
|
||||
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
|
||||
}
|
||||
return resolveCLIReferenceFlatTarget(targets, slotName)
|
||||
}
|
||||
|
||||
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.stage != stage {
|
||||
continue
|
||||
if !laneSelected {
|
||||
return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, selector.LaneID)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
return matches, nil
|
||||
case cliReferenceScopeBinding:
|
||||
laneSelected := false
|
||||
for _, target := range targets {
|
||||
if target.laneID != selector.LaneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if target.stage != selector.Stage {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return nil, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||
}
|
||||
return []selectedReferenceTarget{target}, nil
|
||||
}
|
||||
if !laneSelected {
|
||||
return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
return nil, fmt.Errorf("reference %s target is not selected for lane %q", selector.Stage, selector.LaneID)
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
return nil, fmt.Errorf("reference selector has unknown scope")
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
|
||||
laneSelected := false
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.laneID != laneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
if !laneSelected {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func targetList(targets []selectedReferenceTarget) string {
|
||||
labels := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
labels = append(labels, targetLabel(target))
|
||||
}
|
||||
sort.Strings(labels)
|
||||
return strings.Join(labels, ", ")
|
||||
}
|
||||
|
||||
func targetLabel(target selectedReferenceTarget) string {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
return "chunk"
|
||||
@@ -1753,17 +1839,19 @@ func targetLabel(target selectedReferenceTarget) string {
|
||||
return target.laneID + "." + string(target.stage)
|
||||
}
|
||||
|
||||
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
|
||||
suggestions := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
suggestions = append(suggestions, "chunk."+slotName)
|
||||
continue
|
||||
}
|
||||
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
|
||||
func referenceStageOrder(stage pipeline.ModuleStage) int {
|
||||
switch stage {
|
||||
case pipeline.StageChunk:
|
||||
return 0
|
||||
case pipeline.StageExtract:
|
||||
return 1
|
||||
case pipeline.StageMerge:
|
||||
return 2
|
||||
case pipeline.StageNormalize:
|
||||
return 3
|
||||
default:
|
||||
return 4
|
||||
}
|
||||
sort.Strings(suggestions)
|
||||
return strings.Join(suggestions, " or ")
|
||||
}
|
||||
|
||||
func sortedPipelineIDs(cfg config.Config) []string {
|
||||
|
||||
@@ -264,7 +264,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
||||
}
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
|
||||
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
|
||||
@@ -102,11 +102,12 @@ type TypedNormalizeResult[T any] struct {
|
||||
ModelCandidate *ModelCandidate
|
||||
}
|
||||
|
||||
// Normalize retry diagnostic limits bound module-provided values before the
|
||||
// framework persists them in debug artifacts.
|
||||
// Normalize retry limits bound module-provided control and diagnostic text
|
||||
// before the framework consumes or records it.
|
||||
const (
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryCorrectionGuidanceBytes = 4096
|
||||
)
|
||||
|
||||
// NormalizeRetry asks the framework to retry normalization while retaining a
|
||||
@@ -114,6 +115,7 @@ const (
|
||||
type NormalizeRetry struct {
|
||||
ReasonCode string
|
||||
Message string
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []ProducerDiagnostic
|
||||
}
|
||||
|
||||
|
||||
@@ -30,5 +30,14 @@ func validateNormalizeRetry(retry *contracts.NormalizeRetry) error {
|
||||
if len(retry.Message) > contracts.MaxNormalizeRetryMessageBytes {
|
||||
return errors.New("normalize retry directive message exceeds maximum length")
|
||||
}
|
||||
if !utf8.ValidString(retry.CorrectionGuidance) {
|
||||
return errors.New("normalize retry directive correction guidance has invalid UTF-8")
|
||||
}
|
||||
if retry.CorrectionGuidance != "" && strings.TrimSpace(retry.CorrectionGuidance) == "" {
|
||||
return errors.New("normalize retry directive correction guidance is blank")
|
||||
}
|
||||
if len(retry.CorrectionGuidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
return errors.New("normalize retry directive correction guidance exceeds maximum length")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ func cloneNormalizeRetry(retry *contracts.NormalizeRetry) *contracts.NormalizeRe
|
||||
return &contracts.NormalizeRetry{
|
||||
ReasonCode: retry.ReasonCode,
|
||||
Message: retry.Message,
|
||||
CorrectionGuidance: retry.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(retry.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,9 @@ func (n retryingNotesNormalizer) Normalize(_ context.Context, req contracts.Type
|
||||
|
||||
func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
|
||||
retry := &contracts.NormalizeRetry{
|
||||
ReasonCode: "retryable",
|
||||
Message: "safe fallback available",
|
||||
ReasonCode: "retryable",
|
||||
Message: "safe fallback available",
|
||||
CorrectionGuidance: "return a complete corrected proposal",
|
||||
}
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: "test/retry-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
@@ -41,7 +42,8 @@ func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
retry.Message = "mutated"
|
||||
if result.Retry == nil || result.Retry.Message != "safe fallback available" {
|
||||
retry.CorrectionGuidance = "mutated guidance"
|
||||
if result.Retry == nil || result.Retry.Message != "safe fallback available" || result.Retry.CorrectionGuidance != "return a complete corrected proposal" {
|
||||
t.Fatalf("erased retry result = %#v, want independent retry data", result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ type producerAttemptRequest struct {
|
||||
// the current value as a safe fallback if its shared budget is exhausted.
|
||||
// Artifact-specific adapters are responsible for validating and populating it.
|
||||
type producerRetryDirective struct {
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []contracts.ProducerDiagnostic
|
||||
}
|
||||
|
||||
@@ -59,7 +60,10 @@ func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
||||
if directive == nil {
|
||||
return nil
|
||||
}
|
||||
return &producerRetryDirective{FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics)}
|
||||
return &producerRetryDirective{
|
||||
CorrectionGuidance: directive.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
||||
@@ -204,10 +208,21 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
if output.Retry != nil && number < attemptLimit {
|
||||
correction, err = moduleRetryCorrection(output)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
||||
kind, correction = producerAttemptModuleRetry, nil
|
||||
kind = producerAttemptModuleRetry
|
||||
continue
|
||||
}
|
||||
if output.Retry != nil && output.Retry.CorrectionGuidance != "" {
|
||||
if _, err := moduleRetryCorrection(output); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
}
|
||||
if output.Retry != nil {
|
||||
output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...)
|
||||
}
|
||||
@@ -269,6 +284,23 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
|
||||
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
||||
}
|
||||
|
||||
func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) {
|
||||
if output.Retry == nil || output.Retry.CorrectionGuidance == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if output.Candidate == nil {
|
||||
return nil, errors.New("module retry correction guidance requires a model candidate")
|
||||
}
|
||||
if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||
return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1)
|
||||
}
|
||||
correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct module retry semantic correction: %w", err)
|
||||
}
|
||||
return correction, nil
|
||||
}
|
||||
|
||||
func validateProducerAttemptDiagnostics(output producerAttemptOutput) error {
|
||||
if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil {
|
||||
return fmt.Errorf("producer returned invalid diagnostics: %w", err)
|
||||
|
||||
@@ -218,6 +218,9 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
calls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
|
||||
}
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
||||
}
|
||||
@@ -234,6 +237,58 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback retry", func(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
guidance = "Use only candidate numbers from the supplied candidate list. Return one complete corrected response."
|
||||
)
|
||||
var observed *contracts.SemanticCorrection
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
|
||||
}
|
||||
observed = request.Correction
|
||||
return producerAttemptOutput{Value: "corrected"}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
|
||||
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
|
||||
}
|
||||
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
|
||||
case 2:
|
||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
|
||||
t.Fatalf("second attempt correction = %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
|
||||
case 3:
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "accepted"}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, approveAttempt)
|
||||
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
@@ -248,6 +303,24 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
|
||||
producerCalls := 0
|
||||
validatorCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
validatorCalls++
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
|
||||
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -107,8 +107,12 @@ func (attemptDebugLLM) CompleteStructured(_ context.Context, request contracts.S
|
||||
}
|
||||
|
||||
func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
|
||||
return preparedAttemptDebugPipelineWithChunks(t, 1)
|
||||
}
|
||||
|
||||
func preparedAttemptDebugPipelineWithChunks(t *testing.T, chunkCount int) *PreparedPipeline {
|
||||
t.Helper()
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
prepared := preparedConcurrentPipeline(t, chunkCount)
|
||||
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
|
||||
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
|
||||
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]
|
||||
|
||||
@@ -153,6 +153,60 @@ func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerForwardsModuleRequestedNormalizeCorrection(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
guidance = "Duplicate group 1 must use only supplied candidate numbers. Return one complete corrected response."
|
||||
)
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
var observed *contracts.SemanticCorrection
|
||||
calls := 0
|
||||
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
clone, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
observed = clone
|
||||
}
|
||||
if calls == 1 {
|
||||
return erasedTypedResult{
|
||||
Value: codecNotes{Items: []string{"safe fallback"}},
|
||||
ModelCandidate: attemptCandidate(t, defective),
|
||||
Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: "semantic_proposal_invalid",
|
||||
Message: "operator-facing proposal diagnostic",
|
||||
CorrectionGuidance: guidance,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"corrected"}}}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != 2 || observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("normalize calls = %d correction = %#v", calls, observed)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted output", output)
|
||||
}
|
||||
var retryDebug strings.Builder
|
||||
for _, name := range debug.names() {
|
||||
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
|
||||
retryDebug.Write(debug.json[name])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(retryDebug.String(), `"correction_available":true`) || strings.Contains(retryDebug.String(), guidance) || strings.Contains(retryDebug.String(), defective) {
|
||||
t.Fatalf("retry debug = %s, want safe correction metadata without content", retryDebug.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
const (
|
||||
reasonSentinel = "reason-diagnostic-sentinel"
|
||||
@@ -160,9 +214,11 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
)
|
||||
reasonOverLimit := strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes-len(reasonSentinel)) + reasonSentinel + "x"
|
||||
messageOverLimit := strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
guidanceOverLimit := strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
tests := []struct {
|
||||
name string
|
||||
retry contracts.NormalizeRetry
|
||||
candidate *contracts.ModelCandidate
|
||||
wantError string
|
||||
hiddenValues []string
|
||||
}{
|
||||
@@ -173,6 +229,15 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
Message: strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "accepts correction guidance byte limit with candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes),
|
||||
},
|
||||
candidate: attemptCandidate(t, `{"duplicate_groups":[]}`),
|
||||
},
|
||||
{
|
||||
name: "rejects oversized reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
@@ -209,6 +274,46 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
wantError: "message has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects oversized correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: guidanceOverLimit,
|
||||
},
|
||||
wantError: "correction guidance exceeds maximum length",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects invalid correction guidance UTF-8",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: messageSentinel + string([]byte{0xff}),
|
||||
},
|
||||
wantError: "correction guidance has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: " \t\n ",
|
||||
},
|
||||
wantError: "correction guidance is blank",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects correction guidance without candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: "Return one complete corrected response.",
|
||||
},
|
||||
wantError: "requires a model candidate",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
@@ -233,7 +338,7 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry}, nil
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry, ModelCandidate: tc.candidate}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -427,7 +428,7 @@ func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedAttemptDebugPipelineWithChunks(t, 2)
|
||||
debug := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
|
||||
installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
@@ -449,15 +450,23 @@ func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
module := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
|
||||
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/01-llm-check-attempt-01.json")
|
||||
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/chunk-000001/attempt-01/") {
|
||||
t.Fatalf("module LLM calls = %#v, want extract module call only", module.LLMCalls)
|
||||
validatorResponses := make(map[string]struct{})
|
||||
for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ {
|
||||
chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber)
|
||||
module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json")
|
||||
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json")
|
||||
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") {
|
||||
t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls)
|
||||
}
|
||||
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") {
|
||||
t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls)
|
||||
}
|
||||
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
|
||||
t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls)
|
||||
}
|
||||
validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{}
|
||||
}
|
||||
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/") {
|
||||
t.Fatalf("validator LLM calls = %#v, want validator call only", validator.LLMCalls)
|
||||
}
|
||||
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
|
||||
t.Fatalf("module and validator attempts share LLM call %#v", module.LLMCalls)
|
||||
if len(validatorResponses) != 2 {
|
||||
t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -451,8 +451,8 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
return producerAttemptOutput{}, terminal.record(map[string]any{"output": debugCheckpointArtifact(serializedCandidate)}, fmt.Errorf("normalize lane %q returned invalid retry directive: %w", lane.ID, err))
|
||||
}
|
||||
anotherAttempt := request.Number <= lane.Normalize.Retries
|
||||
attemptValue.retry = map[string]any{"reason_code": result.Retry.ReasonCode, "message": result.Retry.Message, "another_attempt": anotherAttempt, "fallback_accepted": !anotherAttempt}
|
||||
directive = &producerRetryDirective{FallbackDiagnostics: contracts.CloneProducerDiagnostics(result.Retry.FallbackDiagnostics)}
|
||||
attemptValue.retry = map[string]any{"reason_code": result.Retry.ReasonCode, "message": result.Retry.Message, "correction_available": result.Retry.CorrectionGuidance != "", "another_attempt": anotherAttempt, "fallback_accepted": !anotherAttempt}
|
||||
directive = &producerRetryDirective{CorrectionGuidance: result.Retry.CorrectionGuidance, FallbackDiagnostics: contracts.CloneProducerDiagnostics(result.Retry.FallbackDiagnostics)}
|
||||
if anotherAttempt {
|
||||
payload := map[string]any{"output": debugCheckpointArtifact(serializedCandidate), "retry": attemptValue.retry}
|
||||
if debugErr := terminal.record(payload, nil); debugErr != nil {
|
||||
@@ -596,7 +596,12 @@ func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEnt
|
||||
var result contracts.ValidationResult
|
||||
var err error
|
||||
started := time.Now().UTC()
|
||||
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
|
||||
pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)}
|
||||
if target.chunk != nil {
|
||||
pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1))
|
||||
}
|
||||
pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt))
|
||||
attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt)
|
||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
||||
requestTarget := target
|
||||
requestTarget.sourceInput = target.sourceInput.Clone()
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Mira"},{"candidate_id":2,"label":"Captain Mira"}]}`),
|
||||
"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[{"candidate_number":1,"label":"Mira"},{"candidate_number":2,"label":"Captain Mira"}]}`),
|
||||
"transcript": promptkit.Inline(`{"windows":[{"units":[{"unit_id":7,"text":"Mira arrived."}]}]}`),
|
||||
},
|
||||
})
|
||||
@@ -60,15 +60,15 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
protocol := prepared.Messages[1].Content
|
||||
for _, requirement := range []string{"positive integer", "Return IDs only", "do not copy candidate names", "source ranges"} {
|
||||
for _, requirement := range []string{"candidate_number_range", "Return candidate numbers only", "do not copy candidate names", "start_unit_id", "not candidate numbers"} {
|
||||
if !strings.Contains(protocol, requirement) {
|
||||
t.Fatalf("protocol message = %q, want requirement %q", protocol, requirement)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[3].Content, `"candidate_id":1`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) {
|
||||
if !strings.Contains(prepared.Messages[3].Content, `"candidate_number":1`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) {
|
||||
t.Fatalf("candidate message = %q, want only integer candidate material", prepared.Messages[3].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"candidate_id"`) {
|
||||
if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"candidate_number"`) {
|
||||
t.Fatalf("transcript message = %q, want only transcript windows", prepared.Messages[4].Content)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,12 +65,13 @@ const (
|
||||
|
||||
// Result owns the safe plan and neutral diagnostics from one call.
|
||||
type Result struct {
|
||||
disposition ResultDisposition
|
||||
plan Plan
|
||||
issues []Issue
|
||||
discardedGroupCount int
|
||||
candidateMappings []CandidateMapping
|
||||
modelCandidate *contracts.ModelCandidate
|
||||
disposition ResultDisposition
|
||||
plan Plan
|
||||
issues []Issue
|
||||
discardedGroupCount int
|
||||
candidateNumberRange CandidateNumberRange
|
||||
candidateMappings []CandidateMapping
|
||||
modelCandidate *contracts.ModelCandidate
|
||||
}
|
||||
|
||||
// Disposition returns the classified outcome.
|
||||
@@ -85,11 +86,27 @@ func (result Result) Issues() []Issue { return append([]Issue(nil), result.issue
|
||||
// DiscardedGroupCount returns the number of excluded proposal groups.
|
||||
func (result Result) DiscardedGroupCount() int { return result.discardedGroupCount }
|
||||
|
||||
// CandidateMappings returns the request-local handle mapping used for this call.
|
||||
// CandidateNumberRange returns the exact request-local range used for this call.
|
||||
func (result Result) CandidateNumberRange() CandidateNumberRange {
|
||||
return result.candidateNumberRange
|
||||
}
|
||||
|
||||
// CandidateMappings returns the request-local number mapping used for this call.
|
||||
func (result Result) CandidateMappings() []CandidateMapping {
|
||||
return append([]CandidateMapping(nil), result.candidateMappings...)
|
||||
}
|
||||
|
||||
// CorrectionDetails returns bounded model-facing descriptions of the result's
|
||||
// proposal issues using the exact candidate-number range for this call.
|
||||
func (result Result) CorrectionDetails() ([]string, error) {
|
||||
return CorrectionDetails(result.Issues(), result.candidateNumberRange)
|
||||
}
|
||||
|
||||
// CorrectionGuidance returns one bounded replacement request for this result.
|
||||
func (result Result) CorrectionGuidance(additionalDetails ...string) (string, error) {
|
||||
return CorrectionGuidance(result.Issues(), result.candidateNumberRange, additionalDetails...)
|
||||
}
|
||||
|
||||
// ModelCandidate returns an owned copy of the proposal response when a model
|
||||
// completion produced this result.
|
||||
func (result Result) ModelCandidate() *contracts.ModelCandidate {
|
||||
@@ -158,7 +175,10 @@ func (engine *Engine) Reconcile(ctx context.Context, request Request) (Result, e
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: prepare materials: %w", request.StageName, err)
|
||||
}
|
||||
result := Result{candidateMappings: preparation.CandidateMappings()}
|
||||
result := Result{
|
||||
candidateNumberRange: preparation.CandidateNumberRange(),
|
||||
candidateMappings: preparation.CandidateMappings(),
|
||||
}
|
||||
switch preparation.Disposition() {
|
||||
case InsufficientCandidates:
|
||||
result.disposition = SkippedInsufficientCandidates
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestNewEngineValidatesConstruction(t *testing.T) {
|
||||
|
||||
func TestEnginePropagatesRequestAndAssessesResponse(t *testing.T) {
|
||||
client := &recordingReconciliationClient{responses: []ProposalResponse{{DuplicateGroups: []DuplicateGroup{{
|
||||
CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2,
|
||||
CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 2,
|
||||
}}}}}
|
||||
engine := newTestEngine(t, client, DefaultLimits())
|
||||
request := readyEngineRequest()
|
||||
@@ -69,6 +69,9 @@ func TestEnginePropagatesRequestAndAssessesResponse(t *testing.T) {
|
||||
if result.Disposition() != Complete || result.DiscardedGroupCount() != 0 || len(result.Issues()) != 0 {
|
||||
t.Fatalf("result = disposition %v discarded %d issues %#v", result.Disposition(), result.DiscardedGroupCount(), result.Issues())
|
||||
}
|
||||
if got, want := result.CandidateNumberRange(), (CandidateNumberRange{First: 1, Last: 2}); got != want {
|
||||
t.Fatalf("candidate number range = %#v, want %#v", got, want)
|
||||
}
|
||||
groups := result.Plan().Groups()
|
||||
if len(groups) != 1 || !reflect.DeepEqual(groups[0].MemberPositions(), []int{0, 1}) || groups[0].CanonicalPosition() != 1 {
|
||||
t.Fatalf("safe plan = %#v", groups)
|
||||
@@ -105,7 +108,7 @@ func TestEngineClassifiesSemanticAndTransportOutcomes(t *testing.T) {
|
||||
wantError error
|
||||
}{
|
||||
{name: "empty groups complete", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{}}, want: Complete, wantCandidate: true},
|
||||
{name: "discarded proposal retryable", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1}}}, want: RetryableDiscardedProposalGroups, wantDiscard: 1, wantIssues: true, wantCandidate: true},
|
||||
{name: "discarded proposal retryable", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateNumbers: []int{1, 99}, CanonicalCandidateNumber: 1}}}, want: RetryableDiscardedProposalGroups, wantDiscard: 1, wantIssues: true, wantCandidate: true},
|
||||
{name: "invalid structured output retryable", completion: fmt.Errorf("decode response: %w", contracts.ErrInvalidStructuredOutput), want: RetryableInvalidStructuredOutput},
|
||||
{name: "transport failure", completion: transportErr, wantError: transportErr},
|
||||
}
|
||||
@@ -213,8 +216,8 @@ func TestEngineRejectsInvalidInvocationAndHonorsCancellation(t *testing.T) {
|
||||
func TestEngineCallsAreIndependentAndResultsAreOwned(t *testing.T) {
|
||||
client := &recordingReconciliationClient{responses: []ProposalResponse{
|
||||
{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 1},
|
||||
{CandidateIDs: []int{2, 99}, CanonicalCandidateID: 2},
|
||||
{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 1},
|
||||
{CandidateNumbers: []int{2, 99}, CanonicalCandidateNumber: 2},
|
||||
}},
|
||||
{DuplicateGroups: []DuplicateGroup{}},
|
||||
}}
|
||||
@@ -280,7 +283,7 @@ func cloneProposalResponse(response ProposalResponse) ProposalResponse {
|
||||
cloned := ProposalResponse{DuplicateGroups: make([]DuplicateGroup, len(response.DuplicateGroups))}
|
||||
for index, group := range response.DuplicateGroups {
|
||||
cloned.DuplicateGroups[index] = DuplicateGroup{
|
||||
CandidateIDs: append([]int(nil), group.CandidateIDs...), CanonicalCandidateID: group.CanonicalCandidateID,
|
||||
CandidateNumbers: append([]int(nil), group.CandidateNumbers...), CanonicalCandidateNumber: group.CanonicalCandidateNumber,
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
// Policy identifies the framework-owned reconciliation and assessment rules.
|
||||
const Policy = "semantic_reconciliation.v1"
|
||||
const Policy = "semantic_reconciliation.v3"
|
||||
|
||||
var _ contracts.ManifestMetadataProvider = (*Engine)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Engine)(nil)
|
||||
|
||||
@@ -70,18 +70,26 @@ const (
|
||||
LimitExceeded
|
||||
)
|
||||
|
||||
// CandidateMapping relates one model-visible request-local ID to the
|
||||
// CandidateMapping relates one model-visible request-local number to the
|
||||
// corresponding zero-based position in the caller's candidate slice.
|
||||
type CandidateMapping struct {
|
||||
CandidateID int
|
||||
CandidateNumber int
|
||||
CandidatePosition int
|
||||
}
|
||||
|
||||
// CandidateNumberRange is the exact inclusive request-local number range
|
||||
// presented to the model. Ready preparations always use a one-based range.
|
||||
type CandidateNumberRange struct {
|
||||
First int `json:"first"`
|
||||
Last int `json:"last"`
|
||||
}
|
||||
|
||||
// Preparation owns the visible candidate mapping and prompt materials.
|
||||
type Preparation struct {
|
||||
disposition Disposition
|
||||
mappings []CandidateMapping
|
||||
materials contracts.LLMInputSet
|
||||
disposition Disposition
|
||||
candidateNumberRange CandidateNumberRange
|
||||
mappings []CandidateMapping
|
||||
materials contracts.LLMInputSet
|
||||
}
|
||||
|
||||
// Disposition returns the preparation outcome.
|
||||
@@ -94,6 +102,12 @@ func (preparation Preparation) CandidateMappings() []CandidateMapping {
|
||||
return append([]CandidateMapping(nil), preparation.mappings...)
|
||||
}
|
||||
|
||||
// CandidateNumberRange returns the exact request-local range assigned during
|
||||
// preparation. A zero value means that no eligible candidates were present.
|
||||
func (preparation Preparation) CandidateNumberRange() CandidateNumberRange {
|
||||
return preparation.candidateNumberRange
|
||||
}
|
||||
|
||||
// Materials returns independently owned candidate and transcript materials.
|
||||
// It is empty unless Disposition returns Ready.
|
||||
func (preparation Preparation) Materials() contracts.LLMInputSet {
|
||||
@@ -106,13 +120,14 @@ type sourceRange struct {
|
||||
}
|
||||
|
||||
type visibleCandidate struct {
|
||||
CandidateID int `json:"candidate_id"`
|
||||
Label string `json:"label"`
|
||||
SourceRefs []sourceRange `json:"source_refs"`
|
||||
CandidateNumber int `json:"candidate_number"`
|
||||
Label string `json:"label"`
|
||||
SourceRefs []sourceRange `json:"source_refs"`
|
||||
}
|
||||
|
||||
type candidateInput struct {
|
||||
Candidates []visibleCandidate `json:"candidates"`
|
||||
CandidateNumberRange CandidateNumberRange `json:"candidate_number_range"`
|
||||
Candidates []visibleCandidate `json:"candidates"`
|
||||
}
|
||||
|
||||
type transcriptInput struct {
|
||||
@@ -168,17 +183,20 @@ func Prepare(document *source.SourceDocument, candidates []Candidate, limits Lim
|
||||
disposition: InsufficientCandidates,
|
||||
mappings: make([]CandidateMapping, len(prepared)),
|
||||
}
|
||||
if len(prepared) > 0 {
|
||||
result.candidateNumberRange = CandidateNumberRange{First: 1, Last: len(prepared)}
|
||||
}
|
||||
views := make([]visibleCandidate, len(prepared))
|
||||
for index, candidate := range prepared {
|
||||
candidateID := index + 1
|
||||
candidateNumber := index + 1
|
||||
result.mappings[index] = CandidateMapping{
|
||||
CandidateID: candidateID,
|
||||
CandidateNumber: candidateNumber,
|
||||
CandidatePosition: candidate.position,
|
||||
}
|
||||
views[index] = visibleCandidate{
|
||||
CandidateID: candidateID,
|
||||
Label: candidates[candidate.position].Label,
|
||||
SourceRefs: cloneSourceRanges(candidate.references),
|
||||
CandidateNumber: candidateNumber,
|
||||
Label: candidates[candidate.position].Label,
|
||||
SourceRefs: cloneSourceRanges(candidate.references),
|
||||
}
|
||||
}
|
||||
if len(prepared) < 2 {
|
||||
@@ -189,7 +207,7 @@ func Prepare(document *source.SourceDocument, candidates []Candidate, limits Lim
|
||||
return result, nil
|
||||
}
|
||||
|
||||
candidateContent, withinLimit, err := marshalCandidateInput(views, limits.MaximumMaterialBytes)
|
||||
candidateContent, withinLimit, err := marshalCandidateInput(result.candidateNumberRange, views, limits.MaximumMaterialBytes)
|
||||
if err != nil {
|
||||
return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode candidate material: %w", err)
|
||||
}
|
||||
@@ -302,10 +320,14 @@ func coalesceIntervals(intervals []sourceInterval) []sourceInterval {
|
||||
return coalesced
|
||||
}
|
||||
|
||||
func marshalCandidateInput(candidates []visibleCandidate, maximumBytes int) ([]byte, bool, error) {
|
||||
func marshalCandidateInput(numberRange CandidateNumberRange, candidates []visibleCandidate, maximumBytes int) ([]byte, bool, error) {
|
||||
content := make([]byte, 0, min(maximumBytes, 4096))
|
||||
encodedRange, err := json.Marshal(numberRange)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var withinLimit bool
|
||||
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte(`{"candidates":[`))
|
||||
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte(`{"candidate_number_range":`), encodedRange, []byte(`,"candidates":[`))
|
||||
if !withinLimit {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
@@ -47,11 +47,14 @@ func TestPrepareBuildsContiguousCandidatesAndOwnedSourceContext(t *testing.T) {
|
||||
t.Fatalf("Prepare() mutated candidates: %#v", candidates)
|
||||
}
|
||||
if got, want := preparation.CandidateMappings(), []CandidateMapping{
|
||||
{CandidateID: 1, CandidatePosition: 0},
|
||||
{CandidateID: 2, CandidatePosition: 1},
|
||||
{CandidateNumber: 1, CandidatePosition: 0},
|
||||
{CandidateNumber: 2, CandidatePosition: 1},
|
||||
}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("CandidateMappings() = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := preparation.CandidateNumberRange(), (CandidateNumberRange{First: 1, Last: 2}); got != want {
|
||||
t.Fatalf("CandidateNumberRange() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
materials := preparation.Materials()
|
||||
if len(materials) != 2 {
|
||||
@@ -79,12 +82,15 @@ func TestPrepareBuildsContiguousCandidatesAndOwnedSourceContext(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantCandidates := []visibleCandidate{
|
||||
{CandidateID: 1, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}},
|
||||
{CandidateID: 2, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}},
|
||||
{CandidateNumber: 1, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}},
|
||||
{CandidateNumber: 2, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}},
|
||||
}
|
||||
if !reflect.DeepEqual(candidatePayload.Candidates, wantCandidates) {
|
||||
t.Fatalf("candidate payload = %#v, want %#v", candidatePayload.Candidates, wantCandidates)
|
||||
}
|
||||
if got, want := candidatePayload.CandidateNumberRange, (CandidateNumberRange{First: 1, Last: 2}); got != want {
|
||||
t.Fatalf("candidate number range = %#v, want %#v", got, want)
|
||||
}
|
||||
var candidateObjects struct {
|
||||
Candidates []map[string]json.RawMessage `json:"candidates"`
|
||||
}
|
||||
@@ -92,12 +98,12 @@ func TestPrepareBuildsContiguousCandidatesAndOwnedSourceContext(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, candidate := range candidateObjects.Candidates {
|
||||
if len(candidate) != 3 || candidate["candidate_id"] == nil || candidate["label"] == nil || candidate["source_refs"] == nil {
|
||||
if len(candidate) != 3 || candidate["candidate_number"] == nil || candidate["label"] == nil || candidate["source_refs"] == nil {
|
||||
t.Fatalf("model-facing candidate fields = %#v", candidate)
|
||||
}
|
||||
}
|
||||
combined := string(materials[candidateInputName].Content) + string(materials[transcriptInputName].Content)
|
||||
for _, forbidden := range []string{document.ID, "application_entity_id", "private-entity-id"} {
|
||||
for _, forbidden := range []string{document.ID, "application_entity_id", "private-entity-id", `"candidate_id"`, `"candidate_ids"`, `"canonical_candidate_id"`} {
|
||||
if strings.Contains(combined, forbidden) {
|
||||
t.Fatalf("model material leaked %q: %s", forbidden, combined)
|
||||
}
|
||||
@@ -192,11 +198,14 @@ func TestPrepareFiltersUnsafeCandidatesAndCoalescesAdjacentWindows(t *testing.T)
|
||||
t.Fatalf("Prepare() disposition = %v, error = %v", preparation.Disposition(), err)
|
||||
}
|
||||
if got, want := preparation.CandidateMappings(), []CandidateMapping{
|
||||
{CandidateID: 1, CandidatePosition: 0},
|
||||
{CandidateID: 2, CandidatePosition: 1},
|
||||
{CandidateNumber: 1, CandidatePosition: 0},
|
||||
{CandidateNumber: 2, CandidatePosition: 1},
|
||||
}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("CandidateMappings() = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := preparation.CandidateNumberRange(), (CandidateNumberRange{First: 1, Last: 2}); got != want {
|
||||
t.Fatalf("CandidateNumberRange() after filtering = %#v, want %#v", got, want)
|
||||
}
|
||||
var transcript transcriptInput
|
||||
if err := json.Unmarshal(preparation.Materials()[transcriptInputName].Content, &transcript); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -208,7 +217,7 @@ func TestPrepareFiltersUnsafeCandidatesAndCoalescesAdjacentWindows(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := oneCandidate.CandidateMappings(), []CandidateMapping{{CandidateID: 1, CandidatePosition: 0}}; oneCandidate.Disposition() != InsufficientCandidates || !reflect.DeepEqual(got, want) || len(oneCandidate.Materials()) != 0 {
|
||||
if got, want := oneCandidate.CandidateMappings(), []CandidateMapping{{CandidateNumber: 1, CandidatePosition: 0}}; oneCandidate.Disposition() != InsufficientCandidates || !reflect.DeepEqual(got, want) || oneCandidate.CandidateNumberRange() != (CandidateNumberRange{First: 1, Last: 1}) || len(oneCandidate.Materials()) != 0 {
|
||||
t.Fatalf("Prepare(one candidate) = disposition %v, mappings %#v, materials %#v", oneCandidate.Disposition(), got, oneCandidate.Materials())
|
||||
}
|
||||
|
||||
@@ -216,7 +225,7 @@ func TestPrepareFiltersUnsafeCandidatesAndCoalescesAdjacentWindows(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nilPreparation.Disposition() != InsufficientCandidates || len(nilPreparation.CandidateMappings()) != 0 || len(nilPreparation.Materials()) != 0 {
|
||||
if nilPreparation.Disposition() != InsufficientCandidates || len(nilPreparation.CandidateMappings()) != 0 || nilPreparation.CandidateNumberRange() != (CandidateNumberRange{}) || len(nilPreparation.Materials()) != 0 {
|
||||
t.Fatalf("Prepare(nil) = disposition %v, mappings %#v, materials %#v", nilPreparation.Disposition(), nilPreparation.CandidateMappings(), nilPreparation.Materials())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@ package semanticreconcile
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// ProposalResponse is the complete private structured response contract.
|
||||
@@ -10,11 +14,11 @@ type ProposalResponse struct {
|
||||
DuplicateGroups []DuplicateGroup `json:"duplicate_groups"`
|
||||
}
|
||||
|
||||
// DuplicateGroup proposes supplied request-local candidate IDs that may denote
|
||||
// one entity and identifies one supplied member as canonical.
|
||||
// DuplicateGroup proposes supplied request-local candidate numbers that may
|
||||
// denote one entity and identifies one supplied member as canonical.
|
||||
type DuplicateGroup struct {
|
||||
CandidateIDs []int `json:"candidate_ids"`
|
||||
CanonicalCandidateID int `json:"canonical_candidate_id"`
|
||||
CandidateNumbers []int `json:"candidate_numbers"`
|
||||
CanonicalCandidateNumber int `json:"canonical_candidate_number"`
|
||||
}
|
||||
|
||||
// IssueCategory identifies one stable proposal safety failure.
|
||||
@@ -31,6 +35,28 @@ const (
|
||||
IssueOverlappingMember IssueCategory = "overlapping_member"
|
||||
)
|
||||
|
||||
var allIssueCategories = []IssueCategory{
|
||||
IssueMemberNonPositive,
|
||||
IssueMemberUnknown,
|
||||
IssueRepeatedMember,
|
||||
IssueFewerThanTwoMembers,
|
||||
IssueCanonicalNonPositive,
|
||||
IssueCanonicalUnknown,
|
||||
IssueCanonicalNotMember,
|
||||
IssueOverlappingMember,
|
||||
}
|
||||
|
||||
var issueCorrectionProse = map[IssueCategory]string{
|
||||
IssueMemberNonPositive: "Use only candidate numbers in %s.",
|
||||
IssueMemberUnknown: "Remove every candidate number outside %s.",
|
||||
IssueRepeatedMember: "List each candidate number at most once within the duplicate group.",
|
||||
IssueFewerThanTwoMembers: "Include at least two distinct candidate numbers, or omit the duplicate group.",
|
||||
IssueCanonicalNonPositive: "Choose `canonical_candidate_number` from %s.",
|
||||
IssueCanonicalUnknown: "Choose `canonical_candidate_number` from %s.",
|
||||
IssueCanonicalNotMember: "Make `canonical_candidate_number` one of the `candidate_numbers` in the same duplicate group.",
|
||||
IssueOverlappingMember: "Place each candidate number in at most one duplicate group.",
|
||||
}
|
||||
|
||||
// Issue identifies an unsafe proposal category at its original response group
|
||||
// index without prescribing caller diagnostic text.
|
||||
type Issue struct {
|
||||
@@ -38,8 +64,8 @@ type Issue struct {
|
||||
Category IssueCategory
|
||||
}
|
||||
|
||||
// IssueDetails renders stable, domain-neutral proposal diagnostics for an
|
||||
// adapter's retry message.
|
||||
// IssueDetails renders stable, domain-neutral proposal diagnostics for
|
||||
// operators and debug records. Its internal categories are not model guidance.
|
||||
func IssueDetails(issues []Issue) []string {
|
||||
details := make([]string, len(issues))
|
||||
for index, issue := range issues {
|
||||
@@ -48,6 +74,126 @@ func IssueDetails(issues []Issue) []string {
|
||||
return details
|
||||
}
|
||||
|
||||
// CorrectionDetails translates proposal issues into stable model-facing prose.
|
||||
// The response-local group ordinals help the model find the defective group in
|
||||
// the exact response appended to the correction request.
|
||||
func CorrectionDetails(issues []Issue, numberRange CandidateNumberRange) ([]string, error) {
|
||||
if err := validateCorrectionRange(numberRange); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groupsByCategory := make(map[IssueCategory][]int)
|
||||
seen := make(map[IssueCategory]map[int]struct{})
|
||||
for _, issue := range issues {
|
||||
if issue.GroupIndex < 0 {
|
||||
return nil, fmt.Errorf("semantic reconciliation issue group index must not be negative")
|
||||
}
|
||||
if _, exists := issueCorrectionProse[issue.Category]; !exists {
|
||||
return nil, fmt.Errorf("semantic reconciliation issue category %q has no correction guidance", issue.Category)
|
||||
}
|
||||
if seen[issue.Category] == nil {
|
||||
seen[issue.Category] = make(map[int]struct{})
|
||||
}
|
||||
if _, exists := seen[issue.Category][issue.GroupIndex]; exists {
|
||||
continue
|
||||
}
|
||||
seen[issue.Category][issue.GroupIndex] = struct{}{}
|
||||
groupsByCategory[issue.Category] = append(groupsByCategory[issue.Category], issue.GroupIndex)
|
||||
}
|
||||
|
||||
details := make([]string, 0, len(groupsByCategory))
|
||||
includeNamespaceReminder := false
|
||||
for _, category := range allIssueCategories {
|
||||
groups := groupsByCategory[category]
|
||||
if len(groups) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Ints(groups)
|
||||
prose := issueCorrectionProse[category]
|
||||
if issueNeedsCandidateRange(category) {
|
||||
prose = fmt.Sprintf(prose, candidateRangeDescription(numberRange))
|
||||
includeNamespaceReminder = true
|
||||
}
|
||||
details = append(details, fmt.Sprintf("%s: %s", correctionGroupLabel(groups), prose))
|
||||
}
|
||||
if includeNamespaceReminder {
|
||||
details = append(details, "Candidate numbers are the request-local `candidate_number` values in that range. Transcript unit `id` values and evidence `start_unit_id` and `end_unit_id` values are source positions, not candidate numbers.")
|
||||
}
|
||||
return details, nil
|
||||
}
|
||||
|
||||
// CorrectionGuidance builds one bounded request for a complete corrected
|
||||
// proposal. Additional details let a typed owner append a domain rule without
|
||||
// weakening or duplicating the shared protocol guidance.
|
||||
func CorrectionGuidance(issues []Issue, numberRange CandidateNumberRange, additionalDetails ...string) (string, error) {
|
||||
details, err := CorrectionDetails(issues, numberRange)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, detail := range additionalDetails {
|
||||
if !utf8.ValidString(detail) {
|
||||
return "", fmt.Errorf("semantic reconciliation additional correction detail has invalid UTF-8")
|
||||
}
|
||||
detail = strings.TrimSpace(detail)
|
||||
if detail == "" {
|
||||
return "", fmt.Errorf("semantic reconciliation additional correction detail must not be blank")
|
||||
}
|
||||
details = append(details, detail)
|
||||
}
|
||||
if len(details) == 0 {
|
||||
return "", fmt.Errorf("semantic reconciliation correction guidance requires at least one detail")
|
||||
}
|
||||
|
||||
parts := make([]string, 0, len(details)+2)
|
||||
parts = append(parts, "The previous semantic-duplicate proposal was invalid.")
|
||||
parts = append(parts, details...)
|
||||
parts = append(parts, "Return one complete corrected JSON response that follows the original instructions; do not return a patch or commentary.")
|
||||
guidance := strings.Join(parts, " ")
|
||||
if len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
return "", fmt.Errorf("semantic reconciliation correction guidance exceeds maximum length")
|
||||
}
|
||||
return guidance, nil
|
||||
}
|
||||
|
||||
func validateCorrectionRange(numberRange CandidateNumberRange) error {
|
||||
if numberRange.First != 1 || numberRange.Last < 2 {
|
||||
return fmt.Errorf("semantic reconciliation correction candidate-number range must start at 1 and include at least two candidates")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func candidateRangeDescription(numberRange CandidateNumberRange) string {
|
||||
return fmt.Sprintf("the inclusive candidate-number range %d through %d", numberRange.First, numberRange.Last)
|
||||
}
|
||||
|
||||
func issueNeedsCandidateRange(category IssueCategory) bool {
|
||||
switch category {
|
||||
case IssueMemberNonPositive, IssueMemberUnknown, IssueCanonicalNonPositive, IssueCanonicalUnknown:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func correctionGroupLabel(groupIndexes []int) string {
|
||||
const maximumDisplayedGroups = 12
|
||||
displayed := groupIndexes
|
||||
if len(displayed) > maximumDisplayedGroups {
|
||||
displayed = displayed[:maximumDisplayedGroups]
|
||||
}
|
||||
ordinals := make([]string, len(displayed))
|
||||
for index, groupIndex := range displayed {
|
||||
ordinals[index] = fmt.Sprintf("%d", groupIndex+1)
|
||||
}
|
||||
if len(groupIndexes) == 1 {
|
||||
return "Duplicate group " + ordinals[0]
|
||||
}
|
||||
label := "Duplicate groups " + strings.Join(ordinals, ", ")
|
||||
if omitted := len(groupIndexes) - len(displayed); omitted > 0 {
|
||||
label += fmt.Sprintf(", and %d additional affected group(s)", omitted)
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// PlanGroup identifies one validated group using original candidate positions.
|
||||
type PlanGroup struct {
|
||||
memberPositions []int
|
||||
@@ -118,18 +264,18 @@ type assessedGroup struct {
|
||||
conflicting bool
|
||||
}
|
||||
|
||||
// Assess resolves request-local IDs through the retained preparation mapping
|
||||
// and returns only deterministic, non-overlapping groups.
|
||||
// Assess resolves request-local candidate numbers through the retained
|
||||
// preparation mapping and returns only deterministic, non-overlapping groups.
|
||||
func (preparation Preparation) Assess(response ProposalResponse) Assessment {
|
||||
positionsByID := make(map[int]int, len(preparation.mappings))
|
||||
positionsByNumber := make(map[int]int, len(preparation.mappings))
|
||||
for _, mapping := range preparation.mappings {
|
||||
positionsByID[mapping.CandidateID] = mapping.CandidatePosition
|
||||
positionsByNumber[mapping.CandidateNumber] = mapping.CandidatePosition
|
||||
}
|
||||
|
||||
groups := make([]assessedGroup, len(response.DuplicateGroups))
|
||||
owners := make(map[int][]int)
|
||||
for groupIndex, proposal := range response.DuplicateGroups {
|
||||
groups[groupIndex] = assessGroup(proposal, positionsByID)
|
||||
groups[groupIndex] = assessGroup(proposal, positionsByNumber)
|
||||
if !groups[groupIndex].locallyValid {
|
||||
continue
|
||||
}
|
||||
@@ -169,17 +315,17 @@ func (preparation Preparation) Assess(response ProposalResponse) Assessment {
|
||||
return assessment
|
||||
}
|
||||
|
||||
func assessGroup(proposal DuplicateGroup, positionsByID map[int]int) assessedGroup {
|
||||
func assessGroup(proposal DuplicateGroup, positionsByNumber map[int]int) assessedGroup {
|
||||
group := assessedGroup{}
|
||||
seenIDs := make(map[int]struct{}, len(proposal.CandidateIDs))
|
||||
memberPositions := make(map[int]struct{}, len(proposal.CandidateIDs))
|
||||
for _, candidateID := range proposal.CandidateIDs {
|
||||
if _, repeated := seenIDs[candidateID]; repeated {
|
||||
seenNumbers := make(map[int]struct{}, len(proposal.CandidateNumbers))
|
||||
memberPositions := make(map[int]struct{}, len(proposal.CandidateNumbers))
|
||||
for _, candidateNumber := range proposal.CandidateNumbers {
|
||||
if _, repeated := seenNumbers[candidateNumber]; repeated {
|
||||
group.issues = append(group.issues, IssueRepeatedMember)
|
||||
continue
|
||||
}
|
||||
seenIDs[candidateID] = struct{}{}
|
||||
position, category := resolveMember(candidateID, positionsByID)
|
||||
seenNumbers[candidateNumber] = struct{}{}
|
||||
position, category := resolveMember(candidateNumber, positionsByNumber)
|
||||
if category != "" {
|
||||
group.issues = append(group.issues, category)
|
||||
continue
|
||||
@@ -191,7 +337,7 @@ func assessGroup(proposal DuplicateGroup, positionsByID map[int]int) assessedGro
|
||||
group.issues = append(group.issues, IssueFewerThanTwoMembers)
|
||||
}
|
||||
|
||||
canonicalPosition, canonicalCategory := resolveCanonical(proposal.CanonicalCandidateID, positionsByID)
|
||||
canonicalPosition, canonicalCategory := resolveCanonical(proposal.CanonicalCandidateNumber, positionsByNumber)
|
||||
if canonicalCategory != "" {
|
||||
group.issues = append(group.issues, canonicalCategory)
|
||||
} else {
|
||||
@@ -206,22 +352,22 @@ func assessGroup(proposal DuplicateGroup, positionsByID map[int]int) assessedGro
|
||||
return group
|
||||
}
|
||||
|
||||
func resolveMember(candidateID int, positionsByID map[int]int) (int, IssueCategory) {
|
||||
if candidateID <= 0 {
|
||||
func resolveMember(candidateNumber int, positionsByNumber map[int]int) (int, IssueCategory) {
|
||||
if candidateNumber <= 0 {
|
||||
return 0, IssueMemberNonPositive
|
||||
}
|
||||
position, exists := positionsByID[candidateID]
|
||||
position, exists := positionsByNumber[candidateNumber]
|
||||
if !exists {
|
||||
return 0, IssueMemberUnknown
|
||||
}
|
||||
return position, ""
|
||||
}
|
||||
|
||||
func resolveCanonical(candidateID int, positionsByID map[int]int) (int, IssueCategory) {
|
||||
if candidateID <= 0 {
|
||||
func resolveCanonical(candidateNumber int, positionsByNumber map[int]int) (int, IssueCategory) {
|
||||
if candidateNumber <= 0 {
|
||||
return 0, IssueCanonicalNonPositive
|
||||
}
|
||||
position, exists := positionsByID[candidateID]
|
||||
position, exists := positionsByNumber[candidateNumber]
|
||||
if !exists {
|
||||
return 0, IssueCanonicalUnknown
|
||||
}
|
||||
|
||||
@@ -2,16 +2,18 @@ package semanticreconcile
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestAssessProducesAStableOriginalPositionPlan(t *testing.T) {
|
||||
preparation := proposalPreparation(t)
|
||||
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{5, 4}, CanonicalCandidateID: 5},
|
||||
{CandidateIDs: []int{2, 1}, CanonicalCandidateID: 2},
|
||||
{CandidateNumbers: []int{5, 4}, CanonicalCandidateNumber: 5},
|
||||
{CandidateNumbers: []int{2, 1}, CanonicalCandidateNumber: 2},
|
||||
}}
|
||||
assessment := preparation.Assess(response)
|
||||
want := []planGroupSnapshot{
|
||||
@@ -26,8 +28,8 @@ func TestAssessProducesAStableOriginalPositionPlan(t *testing.T) {
|
||||
}
|
||||
|
||||
reordered := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
||||
{CandidateIDs: []int{4, 5}, CanonicalCandidateID: 5},
|
||||
{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 2},
|
||||
{CandidateNumbers: []int{4, 5}, CanonicalCandidateNumber: 5},
|
||||
}})
|
||||
if got := snapshotPlan(reordered.Plan()); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("reordered plan = %#v, want %#v", got, want)
|
||||
@@ -50,6 +52,79 @@ func TestIssueDetailsPreservesIssueOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionGuidanceCoversEveryIssueCategoryWithoutExposingInternalLabels(t *testing.T) {
|
||||
numberRange := CandidateNumberRange{First: 1, Last: 8}
|
||||
if len(issueCorrectionProse) != len(allIssueCategories) {
|
||||
t.Fatalf("correction prose entries = %d, categories = %d", len(issueCorrectionProse), len(allIssueCategories))
|
||||
}
|
||||
issues := make([]Issue, len(allIssueCategories))
|
||||
seen := make(map[IssueCategory]struct{}, len(allIssueCategories))
|
||||
for index, category := range allIssueCategories {
|
||||
if _, duplicate := seen[category]; duplicate {
|
||||
t.Fatalf("duplicate authoritative issue category %q", category)
|
||||
}
|
||||
seen[category] = struct{}{}
|
||||
if strings.TrimSpace(issueCorrectionProse[category]) == "" {
|
||||
t.Fatalf("issue category %q has no model-facing prose", category)
|
||||
}
|
||||
issues[index] = Issue{GroupIndex: index, Category: category}
|
||||
}
|
||||
|
||||
details, err := CorrectionDetails(issues, numberRange)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionDetails() error = %v", err)
|
||||
}
|
||||
guidance, err := CorrectionGuidance(issues, numberRange)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionGuidance() error = %v", err)
|
||||
}
|
||||
if len(details) != len(allIssueCategories)+1 || !strings.Contains(guidance, "Duplicate group 1") || !strings.Contains(guidance, "candidate-number range 1 through 8") || !strings.Contains(guidance, "start_unit_id") || !strings.Contains(guidance, "complete corrected JSON response") || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
t.Fatalf("correction details = %#v guidance = %q", details, guidance)
|
||||
}
|
||||
for _, category := range allIssueCategories {
|
||||
if strings.Contains(guidance, string(category)) {
|
||||
t.Fatalf("model guidance exposed internal category %q: %q", category, guidance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionDetailsDeduplicatesAndBoundsAffectedGroupLists(t *testing.T) {
|
||||
numberRange := CandidateNumberRange{First: 1, Last: 256}
|
||||
issues := make([]Issue, 0, 257)
|
||||
for group := 0; group < 256; group++ {
|
||||
issues = append(issues, Issue{GroupIndex: group, Category: IssueMemberUnknown})
|
||||
}
|
||||
issues = append(issues, Issue{GroupIndex: 0, Category: IssueMemberUnknown})
|
||||
|
||||
details, err := CorrectionDetails(issues, numberRange)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionDetails() error = %v", err)
|
||||
}
|
||||
if len(details) != 2 || !strings.Contains(details[0], "additional affected group") || strings.Count(details[0], "Duplicate groups") != 1 || !strings.Contains(details[1], "source positions") {
|
||||
t.Fatalf("CorrectionDetails() = %#v, want one bounded grouped detail and one namespace reminder", details)
|
||||
}
|
||||
guidance, err := CorrectionGuidance(issues, numberRange)
|
||||
if err != nil || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
t.Fatalf("CorrectionGuidance() = %q, %v", guidance, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionDetailsRejectsUnknownOrInvalidIssues(t *testing.T) {
|
||||
for _, issues := range [][]Issue{
|
||||
{{GroupIndex: 0, Category: "future_unmapped_category"}},
|
||||
{{GroupIndex: -1, Category: IssueMemberUnknown}},
|
||||
} {
|
||||
if details, err := CorrectionDetails(issues, CandidateNumberRange{First: 1, Last: 2}); err == nil || details != nil {
|
||||
t.Fatalf("CorrectionDetails(%#v) = %#v, %v; want fail-closed error", issues, details, err)
|
||||
}
|
||||
}
|
||||
for _, numberRange := range []CandidateNumberRange{{}, {First: 0, Last: 2}, {First: 1, Last: 1}} {
|
||||
if details, err := CorrectionDetails([]Issue{{GroupIndex: 0, Category: IssueMemberUnknown}}, numberRange); err == nil || details != nil {
|
||||
t.Fatalf("CorrectionDetails(range %#v) = %#v, %v; want fail-closed error", numberRange, details, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) {
|
||||
preparation := proposalPreparation(t)
|
||||
tests := []struct {
|
||||
@@ -57,15 +132,15 @@ func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) {
|
||||
group DuplicateGroup
|
||||
category IssueCategory
|
||||
}{
|
||||
{name: "zero member", group: DuplicateGroup{CandidateIDs: []int{0, 2}, CanonicalCandidateID: 2}, category: IssueMemberNonPositive},
|
||||
{name: "negative member", group: DuplicateGroup{CandidateIDs: []int{-1, 2}, CanonicalCandidateID: 2}, category: IssueMemberNonPositive},
|
||||
{name: "unknown member", group: DuplicateGroup{CandidateIDs: []int{99, 2}, CanonicalCandidateID: 2}, category: IssueMemberUnknown},
|
||||
{name: "repeated member", group: DuplicateGroup{CandidateIDs: []int{1, 1}, CanonicalCandidateID: 1}, category: IssueRepeatedMember},
|
||||
{name: "too small", group: DuplicateGroup{CandidateIDs: []int{1}, CanonicalCandidateID: 1}, category: IssueFewerThanTwoMembers},
|
||||
{name: "zero canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 0}, category: IssueCanonicalNonPositive},
|
||||
{name: "negative canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: -1}, category: IssueCanonicalNonPositive},
|
||||
{name: "unknown canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 99}, category: IssueCanonicalUnknown},
|
||||
{name: "canonical not member", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 3}, category: IssueCanonicalNotMember},
|
||||
{name: "zero member", group: DuplicateGroup{CandidateNumbers: []int{0, 2}, CanonicalCandidateNumber: 2}, category: IssueMemberNonPositive},
|
||||
{name: "negative member", group: DuplicateGroup{CandidateNumbers: []int{-1, 2}, CanonicalCandidateNumber: 2}, category: IssueMemberNonPositive},
|
||||
{name: "unknown member", group: DuplicateGroup{CandidateNumbers: []int{99, 2}, CanonicalCandidateNumber: 2}, category: IssueMemberUnknown},
|
||||
{name: "repeated member", group: DuplicateGroup{CandidateNumbers: []int{1, 1}, CanonicalCandidateNumber: 1}, category: IssueRepeatedMember},
|
||||
{name: "too small", group: DuplicateGroup{CandidateNumbers: []int{1}, CanonicalCandidateNumber: 1}, category: IssueFewerThanTwoMembers},
|
||||
{name: "zero canonical", group: DuplicateGroup{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 0}, category: IssueCanonicalNonPositive},
|
||||
{name: "negative canonical", group: DuplicateGroup{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: -1}, category: IssueCanonicalNonPositive},
|
||||
{name: "unknown canonical", group: DuplicateGroup{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 99}, category: IssueCanonicalUnknown},
|
||||
{name: "canonical not member", group: DuplicateGroup{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 3}, category: IssueCanonicalNotMember},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -83,9 +158,9 @@ func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) {
|
||||
func TestAssessDiscardsEveryOverlappingGroupAndRetainsIndependentGroups(t *testing.T) {
|
||||
preparation := proposalPreparation(t)
|
||||
assessment := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 1},
|
||||
{CandidateIDs: []int{2, 3}, CanonicalCandidateID: 2},
|
||||
{CandidateIDs: []int{4, 5}, CanonicalCandidateID: 5},
|
||||
{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 1},
|
||||
{CandidateNumbers: []int{2, 3}, CanonicalCandidateNumber: 2},
|
||||
{CandidateNumbers: []int{4, 5}, CanonicalCandidateNumber: 5},
|
||||
}})
|
||||
wantPlan := []planGroupSnapshot{{members: []int{5, 6}, canonical: 6}}
|
||||
if got := snapshotPlan(assessment.Plan()); !reflect.DeepEqual(got, wantPlan) {
|
||||
@@ -100,8 +175,8 @@ func TestAssessDiscardsEveryOverlappingGroupAndRetainsIndependentGroups(t *testi
|
||||
}
|
||||
|
||||
invalidAndSafe := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1},
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
||||
{CandidateNumbers: []int{1, 99}, CanonicalCandidateNumber: 1},
|
||||
{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 2},
|
||||
}})
|
||||
wantPlan = []planGroupSnapshot{{members: []int{0, 2}, canonical: 2}}
|
||||
if got := snapshotPlan(invalidAndSafe.Plan()); !reflect.DeepEqual(got, wantPlan) {
|
||||
@@ -115,13 +190,13 @@ func TestAssessDiscardsEveryOverlappingGroupAndRetainsIndependentGroups(t *testi
|
||||
func TestAssessmentAccessorsAndInputsDoNotShareRetainedState(t *testing.T) {
|
||||
preparation := proposalPreparation(t)
|
||||
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
||||
{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 2},
|
||||
}}
|
||||
assessment := preparation.Assess(response)
|
||||
want := snapshotPlan(assessment.Plan())
|
||||
|
||||
response.DuplicateGroups[0].CandidateIDs[0] = 99
|
||||
response.DuplicateGroups[0].CanonicalCandidateID = 99
|
||||
response.DuplicateGroups[0].CandidateNumbers[0] = 99
|
||||
response.DuplicateGroups[0].CanonicalCandidateNumber = 99
|
||||
plan := assessment.Plan()
|
||||
groups := plan.Groups()
|
||||
members := groups[0].MemberPositions()
|
||||
@@ -132,7 +207,7 @@ func TestAssessmentAccessorsAndInputsDoNotShareRetainedState(t *testing.T) {
|
||||
}
|
||||
|
||||
invalid := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{{
|
||||
CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1,
|
||||
CandidateNumbers: []int{1, 99}, CanonicalCandidateNumber: 1,
|
||||
}}})
|
||||
issues := invalid.Issues()
|
||||
issues[0].GroupIndex = 99
|
||||
|
||||
@@ -44,20 +44,21 @@ func TestResponseSchemaAcceptsOnlyTheIntegerProposalShape(t *testing.T) {
|
||||
valid bool
|
||||
}{
|
||||
{name: "empty proposal", value: map[string]any{"duplicate_groups": []any{}}, valid: true},
|
||||
{name: "valid group", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 1}}}, valid: true},
|
||||
{name: "semantic canonical mismatch", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 3}}}, valid: true},
|
||||
{name: "valid group", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}, "canonical_candidate_number": 1}}}, valid: true},
|
||||
{name: "semantic canonical mismatch", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}, "canonical_candidate_number": 3}}}, valid: true},
|
||||
{name: "missing proposal", value: map[string]any{}, valid: false},
|
||||
{name: "unknown top-level field", value: map[string]any{"duplicate_groups": []any{}, "extra": true}, valid: false},
|
||||
{name: "missing members", value: map[string]any{"duplicate_groups": []any{map[string]any{"canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "missing canonical", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}}}}, valid: false},
|
||||
{name: "unknown group field", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 1, "name": "replacement"}}}, valid: false},
|
||||
{name: "too few members", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1}, "canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "semantic repeated members", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 1}, "canonical_candidate_id": 1}}}, valid: true},
|
||||
{name: "zero member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{0, 1}, "canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "negative member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{-1, 1}, "canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "non-integer member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2.5}, "canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "zero canonical", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 0}}}, valid: false},
|
||||
{name: "contextual selectors", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 1, "source_refs": []any{}}}}, valid: false},
|
||||
{name: "missing members", value: map[string]any{"duplicate_groups": []any{map[string]any{"canonical_candidate_number": 1}}}, valid: false},
|
||||
{name: "missing canonical", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}}}}, valid: false},
|
||||
{name: "unknown group field", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}, "canonical_candidate_number": 1, "name": "replacement"}}}, valid: false},
|
||||
{name: "former candidate ID fields", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_ids": []any{1, 2}, "canonical_candidate_id": 1}}}, valid: false},
|
||||
{name: "too few members", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1}, "canonical_candidate_number": 1}}}, valid: false},
|
||||
{name: "semantic repeated members", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 1}, "canonical_candidate_number": 1}}}, valid: true},
|
||||
{name: "zero member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{0, 1}, "canonical_candidate_number": 1}}}, valid: false},
|
||||
{name: "negative member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{-1, 1}, "canonical_candidate_number": 1}}}, valid: false},
|
||||
{name: "non-integer member", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2.5}, "canonical_candidate_number": 1}}}, valid: false},
|
||||
{name: "zero canonical", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}, "canonical_candidate_number": 0}}}, valid: false},
|
||||
{name: "contextual selectors", value: map[string]any{"duplicate_groups": []any{map[string]any{"candidate_numbers": []any{1, 2}, "canonical_candidate_number": 1, "source_refs": []any{}}}}, valid: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -72,7 +73,7 @@ func TestResponseSchemaAcceptsOnlyTheIntegerProposalShape(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
content := []byte(`{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`)
|
||||
content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`)
|
||||
if err := validateAgainstSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -80,7 +81,7 @@ func TestResponseSchemaAcceptsOnlyTheIntegerProposalShape(t *testing.T) {
|
||||
if err := json.Unmarshal(content, &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2}}}
|
||||
want := ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateNumbers: []int{1, 2}, CanonicalCandidateNumber: 2}}}
|
||||
if !reflect.DeepEqual(response, want) {
|
||||
t.Fatalf("decoded response = %#v, want %#v", response, want)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func canonicalizeCombatTurn(turn *combatTurnResponse, order shared.SourceRefOrde
|
||||
if turn == nil {
|
||||
return 0, false
|
||||
}
|
||||
refs := order.Canonicalize(canonicalSourceRefs(turn.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(turn.SourceRefs, sourceID)))
|
||||
turn.SourceRefs = combatResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/combat-turns"
|
||||
mappingPolicy = "dnd.combat_turns.extract_mapping.v2"
|
||||
mappingPolicy = "dnd.combat_turns.extract_mapping.v3"
|
||||
sceneGatePolicy = "dnd.combat_turns.scene_gate.v1"
|
||||
)
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Actor: "Aria", TurnKind: "reaction", SourceRefs: []combatSourceRefResponse{
|
||||
{StartUnitID: 10, EndUnitID: 10},
|
||||
{StartUnitID: 10, EndUnitID: 10},
|
||||
{StartUnitID: 2, EndUnitID: 10},
|
||||
{StartUnitID: 2, EndUnitID: 10},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -42,7 +42,7 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
|
||||
if got := []string{result.Value.CombatTurns[0].Actor, result.Value.CombatTurns[1].Actor, result.Value.CombatTurns[2].Actor}; !reflect.DeepEqual(got, []string{"Aria", "Borin", "Unknown"}) {
|
||||
t.Fatalf("actor order = %#v, want source-position order with invalid evidence last", got)
|
||||
}
|
||||
wantRefs := []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 10}}
|
||||
wantRefs := []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 2}}
|
||||
if !reflect.DeepEqual(result.Value.CombatTurns[0].SourceRefs, wantRefs) {
|
||||
t.Fatalf("canonical refs = %#v, want %#v", result.Value.CombatTurns[0].SourceRefs, wantRefs)
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
|
||||
func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
|
||||
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
|
||||
{
|
||||
Actor: " ", TurnKind: "unsupported",
|
||||
Actor: " ", TurnKind: "turn",
|
||||
SourceRefs: []combatSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
|
||||
},
|
||||
}}}
|
||||
@@ -75,7 +75,7 @@ func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
|
||||
t.Fatalf("Extract() error = %v, want nil for candidate values", err)
|
||||
}
|
||||
turn := result.Value.CombatTurns[0]
|
||||
if turn.Actor != " " || turn.TurnKind != "unsupported" {
|
||||
if turn.Actor != " " || turn.TurnKind != "turn" {
|
||||
t.Fatalf("invalid turn fields = %#v, want preserved candidate values", turn)
|
||||
}
|
||||
if turn.SourceRefs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesPrivateCombatShape(t *testing.T) {
|
||||
@@ -37,7 +39,7 @@ func TestLoadResponseSchemaUsesPrivateCombatShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *testing.T) {
|
||||
func TestResponseSchemaLeavesNonCategoricalSemanticsToDeterministicValidators(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -45,7 +47,6 @@ func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *tes
|
||||
semanticCandidate := validCombatResponse()
|
||||
turn := semanticCandidate["combat_turns"].([]any)[0].(map[string]any)
|
||||
turn["actor"] = ""
|
||||
turn["turn_kind"] = "unsupported"
|
||||
ref := turn["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
@@ -54,7 +55,7 @@ func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *tes
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("private schema rejected validator-owned semantics: %v", err)
|
||||
t.Fatalf("private schema rejected non-categorical validator-owned semantics: %v", err)
|
||||
}
|
||||
turn["source_refs"] = []any{}
|
||||
content, err = json.Marshal(semanticCandidate)
|
||||
@@ -79,6 +80,7 @@ func TestResponseSchemaRetainsStructuralBoundary(t *testing.T) {
|
||||
{name: "wrong actor type", mutate: func(turn map[string]any) { turn["actor"] = 1 }},
|
||||
{name: "unknown field", mutate: func(turn map[string]any) { turn["unexpected"] = true }},
|
||||
{name: "missing source refs", mutate: func(turn map[string]any) { delete(turn, "source_refs") }},
|
||||
{name: "unsupported turn kind", mutate: func(turn map[string]any) { turn["turn_kind"] = "unsupported" }},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
candidate := validCombatResponse()
|
||||
@@ -94,6 +96,24 @@ func TestResponseSchemaRetainsStructuralBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedTurnKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.CombatTurnKindTurn), string(dnd.CombatTurnKindReaction), string(dnd.CombatTurnKindLegendaryAction), string(dnd.CombatTurnKindLairAction), string(dnd.CombatTurnKindOther)} {
|
||||
candidate := validCombatResponse()
|
||||
candidate["combat_turns"].([]any)[0].(map[string]any)["turn_kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported turn kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/enemy-events"
|
||||
mappingPolicy = "dnd.enemy_events.extract_mapping.v1"
|
||||
mappingPolicy = "dnd.enemy_events.extract_mapping.v2"
|
||||
sceneGatePolicy = "dnd.enemy_events.scene_gate.v1"
|
||||
)
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
|
||||
func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
|
||||
client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{
|
||||
{Name: "Ashfang", Kind: "killed", SourceRefs: []enemySourceRefResponse{{StartUnitID: 4, EndUnitID: 4}}},
|
||||
{Name: "Ashfang", Kind: "killed", SourceRefs: []enemySourceRefResponse{{StartUnitID: 4, EndUnitID: 3}}},
|
||||
{Name: "Ashfang", Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 1, EndUnitID: 1}, {StartUnitID: 1, EndUnitID: 1}}},
|
||||
{Name: "Orcs", Kind: "fled", SourceRefs: []enemySourceRefResponse{{StartUnitID: 3, EndUnitID: 3}}},
|
||||
{Name: "One orc", Kind: "captured", SourceRefs: []enemySourceRefResponse{{StartUnitID: 2, EndUnitID: 2}}},
|
||||
@@ -31,14 +31,17 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
|
||||
dnd.EnemyEventKindEngaged,
|
||||
dnd.EnemyEventKindCaptured,
|
||||
dnd.EnemyEventKindIncapacitated,
|
||||
dnd.EnemyEventKindFled,
|
||||
dnd.EnemyEventKindKilled,
|
||||
dnd.EnemyEventKindFled,
|
||||
}) {
|
||||
t.Fatalf("event order = %#v", got)
|
||||
}
|
||||
if refs := result.Value.Events[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1}}) {
|
||||
t.Fatalf("canonical source refs = %#v", refs)
|
||||
}
|
||||
if refs := result.Value.Events[3].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "combat-session", StartUnitID: 3, EndUnitID: 4}}) {
|
||||
t.Fatalf("reversed source refs = %#v", refs)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
@@ -56,14 +59,14 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
|
||||
|
||||
func TestExtractPreservesSemanticCandidatesAndResponseOwnership(t *testing.T) {
|
||||
client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{{
|
||||
Name: " ", Kind: "unsupported", SourceRefs: []enemySourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
|
||||
Name: " ", Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
|
||||
}}}}
|
||||
result, err := newEnemyExtractor(t, client).Extract(context.Background(), enemyExtractionRequest(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event := result.Value.Events[0]
|
||||
if event.Name != " " || event.Kind != "unsupported" || event.SourceRefs[0] != (source.SourceRef{SourceID: "combat-session", StartUnitID: 99}) {
|
||||
if event.Name != " " || event.Kind != "engaged" || event.SourceRefs[0] != (source.SourceRef{SourceID: "combat-session", StartUnitID: 99}) {
|
||||
t.Fatalf("semantic candidate = %#v", event)
|
||||
}
|
||||
result.Value.Events[0].SourceRefs[0].StartUnitID = 7
|
||||
|
||||
@@ -20,7 +20,7 @@ func canonicalEnemyEventList(response extractionResponse, order shared.SourceRef
|
||||
}
|
||||
ordered := make([]orderedEnemyEvent, len(response.Events))
|
||||
for index, event := range response.Events {
|
||||
refs := order.Canonicalize(sourceRefs(event.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(sourceRefs(event.SourceRefs, sourceID)))
|
||||
earliest, hasEvidence := order.EarliestValid(refs)
|
||||
ordered[index] = orderedEnemyEvent{
|
||||
value: dnd.EnemyEvent{
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
|
||||
@@ -28,13 +30,12 @@ func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
|
||||
semantic := validEnemyResponse()
|
||||
event := semantic["events"].([]any)[0].(map[string]any)
|
||||
event["name"] = ""
|
||||
event["kind"] = "unsupported"
|
||||
ref := event["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semantic)
|
||||
if err != nil || validateEnemySchema(content, schema.JSONSchema) != nil {
|
||||
t.Fatalf("validator-owned semantics were rejected: %v", err)
|
||||
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(event map[string]any) { delete(event, "name") },
|
||||
func(event map[string]any) { event["kind"] = 1 },
|
||||
func(event map[string]any) { event["kind"] = "unsupported" },
|
||||
func(event map[string]any) { event["unexpected"] = true },
|
||||
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
|
||||
} {
|
||||
@@ -70,6 +72,24 @@ func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.EnemyEventKindEngaged), string(dnd.EnemyEventKindKilled), string(dnd.EnemyEventKindFled), string(dnd.EnemyEventKindCaptured), string(dnd.EnemyEventKindIncapacitated)} {
|
||||
candidate := validEnemyResponse()
|
||||
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validEnemyResponse() map[string]any {
|
||||
return map[string]any{"events": []any{map[string]any{
|
||||
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
|
||||
@@ -26,7 +26,7 @@ func canonicalItemOccurrenceList(response extractionResponse, order shared.Sourc
|
||||
if !found {
|
||||
return dnd.ItemOccurrenceList{}, fmt.Errorf("occurrences[%d].name is not in the item registry", index)
|
||||
}
|
||||
refs := order.Canonicalize(itemOccurrenceSourceRefs(occurrence.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(itemOccurrenceSourceRefs(occurrence.SourceRefs, sourceID)))
|
||||
earliest, hasEvidence := order.EarliestValid(refs)
|
||||
ordered[index] = orderedItemOccurrence{
|
||||
value: dnd.ItemOccurrence{
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
ItemRegistryMaxBytes = itemregistry.MaxBytes
|
||||
)
|
||||
|
||||
const mappingPolicy = "dnd.item_occurrences.extract_mapping.v2"
|
||||
const mappingPolicy = "dnd.item_occurrences.extract_mapping.v3"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
|
||||
@@ -6,12 +6,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
itemoccurrenceshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
|
||||
)
|
||||
|
||||
func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
|
||||
id := itemidentity.DeriveID("Torch")
|
||||
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
|
||||
client := &fakeItemOccurrencesLLMClient{content: append([]byte(nil), rawResponse...)}
|
||||
req := extractionRequest()
|
||||
req.References = itemRegistryReferences(t)
|
||||
@@ -27,7 +29,7 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
|
||||
if len(result.Value.Occurrences) != 1 || result.Value.Occurrences[0].ItemID != id || result.Value.Occurrences[0].Name != "Torch" {
|
||||
t.Fatalf("occurrences = %#v", result.Value.Occurrences)
|
||||
}
|
||||
if refs := result.Value.Occurrences[0].SourceRefs; len(refs) != 1 || refs[0].SourceID != req.Source.ID || refs[0].StartUnitID != 1 || refs[0].EndUnitID != 1 {
|
||||
if refs := result.Value.Occurrences[0].SourceRefs; len(refs) != 1 || refs[0].SourceID != req.Source.ID || refs[0].StartUnitID != 1 || refs[0].EndUnitID != 2 {
|
||||
t.Fatalf("occurrence evidence = %#v, want current-source unit range", refs)
|
||||
}
|
||||
request := client.requests[0]
|
||||
@@ -146,3 +148,46 @@ func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) {
|
||||
t.Fatalf("unexpected prompt input: %#v", client.requests[0].Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCarriesRejectedHolderGuidanceIntoCorrectedAttempt(t *testing.T) {
|
||||
references := itemRegistryReferences(t)
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
defectiveResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
|
||||
defective, err := newExtractor(t, &fakeItemOccurrencesLLMClient{content: defectiveResponse}, references).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validation, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: defective.Value})
|
||||
if err != nil || validation.Approved {
|
||||
t.Fatalf("holder validation = %#v, %v; want rejection", validation, err)
|
||||
}
|
||||
for _, fragment := range []string{"For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
|
||||
if !strings.Contains(validation.CorrectionGuidance, fragment) {
|
||||
t.Fatalf("CorrectionGuidance = %q, want %q", validation.CorrectionGuidance, fragment)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{defective.Value.Occurrences[0].ItemID, "item_id", "sha256", itemoccurrenceshape.Key, itemoccurrenceshape.ReasonCode} {
|
||||
if strings.Contains(validation.CorrectionGuidance, forbidden) {
|
||||
t.Fatalf("CorrectionGuidance leaked implementation identifier %q: %q", forbidden, validation.CorrectionGuidance)
|
||||
}
|
||||
}
|
||||
correction, err := contracts.NewSemanticCorrection(defective.ModelCandidate.Response, validation.CorrectionGuidance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
correctedResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
|
||||
client := &fakeItemOccurrencesLLMClient{content: correctedResponse}
|
||||
req.Correction = correction
|
||||
corrected, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: corrected.Value})
|
||||
if err != nil || !accepted.Approved {
|
||||
t.Fatalf("corrected holder validation = %#v, %v; want approval", accepted, err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Correction == nil || string(client.requests[0].Correction.AssistantResponse) != string(defectiveResponse) || client.requests[0].Correction.UserGuidance != validation.CorrectionGuidance {
|
||||
t.Fatalf("corrected request = %#v, want byte-faithful defective response and semantic guidance", client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +58,17 @@ func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
|
||||
t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content)
|
||||
}
|
||||
}
|
||||
for _, holderRule := range []string{
|
||||
"| `discovered` | `null` | `null` |",
|
||||
"| `acquired` | `null` | `party` or the named party member gaining possession |",
|
||||
"| `lost` | `party` or the named party member losing possession | `null` |",
|
||||
"| `consumed` | `party` or the named party member consuming the item | `null` |",
|
||||
"| `transferred` | one named party member | a different named party member |",
|
||||
} {
|
||||
if !strings.Contains(content, holderRule) {
|
||||
t.Fatalf("prepared prompt does not include holder rule %q", holderRule)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
|
||||
}
|
||||
valid := map[string]any{"occurrences": []any{
|
||||
map[string]any{
|
||||
"name": "", "kind": "unsupported", "quantity": 0, "from": "party", "to": "Party",
|
||||
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
|
||||
},
|
||||
map[string]any{
|
||||
@@ -41,6 +41,7 @@ func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
|
||||
{"missing occurrences", map[string]any{}},
|
||||
{"missing occurrence name", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "name")}}},
|
||||
{"missing nullable field", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "quantity")}}},
|
||||
{"unsupported kind", map[string]any{"occurrences": []any{withField(responseOccurrence(), "kind", "unsupported")}}},
|
||||
{"opaque item identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "item_id", "item:sha256:opaque")}}},
|
||||
{"unknown occurrence field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "extra", true)}}},
|
||||
{"segment-named range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
|
||||
|
||||
@@ -43,7 +43,7 @@ func canonicalizeItem(item *itemResponse, order shared.SourceRefOrder, sourceID
|
||||
return 0, false
|
||||
}
|
||||
item.Name = identity.NormalizeDisplay(item.Name)
|
||||
refs := order.Canonicalize(canonicalSourceRefs(item.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(item.SourceRefs, sourceID)))
|
||||
item.SourceRefs = itemResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/item-registry"
|
||||
mappingPolicy = "dnd.item_registry.extract_mapping.v1"
|
||||
mappingPolicy = "dnd.item_registry.extract_mapping.v2"
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{"chunks", "source.transcript"}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func TestExtractMapsItemsWithOwnedEvidenceAndDeterministicOrder(t *testing.T) {
|
||||
client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{
|
||||
{Name: " Gold Pieces ", SourceRefs: responseSourceRefs(3, 3)},
|
||||
{Name: " Gold Pieces ", SourceRefs: responseSourceRefs(3, 2)},
|
||||
{Name: "Star Compass", SourceRefs: []itemSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 1}, {StartUnitID: 1, EndUnitID: 1}}},
|
||||
}}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
@@ -25,7 +25,7 @@ func TestExtractMapsItemsWithOwnedEvidenceAndDeterministicOrder(t *testing.T) {
|
||||
refs := []source.SourceRef{{SourceID: "session-items", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-items", StartUnitID: 2, EndUnitID: 2}}
|
||||
want := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{ID: identity.DeriveID("Star Compass"), Name: "Star Compass", SourceRefs: refs},
|
||||
{ID: identity.DeriveID("Gold Pieces"), Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: "session-items", StartUnitID: 3, EndUnitID: 3}}},
|
||||
{ID: identity.DeriveID("Gold Pieces"), Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: "session-items", StartUnitID: 2, EndUnitID: 3}}},
|
||||
}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
|
||||
@@ -27,7 +27,7 @@ func canonicalOccurrenceList(response extractionResponse, order shared.SourceRef
|
||||
if !ok {
|
||||
return dnd.LocationOccurrenceList{}, fmt.Errorf("occurrence %d does not match a supplied location selector", index)
|
||||
}
|
||||
refs := order.Canonicalize(canonicalSourceRefs(occurrence.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(occurrence.SourceRefs, sourceID)))
|
||||
earliest, hasEvidence := order.EarliestValid(refs)
|
||||
ordered[index] = orderedOccurrence{value: dnd.LocationOccurrence{
|
||||
LocationID: location.ID,
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/location-occurrences"
|
||||
mappingPolicy = "dnd.location_occurrences.extract_mapping.v2"
|
||||
mappingPolicy = "dnd.location_occurrences.extract_mapping.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestExtractMapsKindsOrdersOccurrencesAndPreservesIndependentFacts(t *testin
|
||||
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{
|
||||
{Name: second.Name, RegistryRefs: registryRefs(second), Kind: "mentioned", SourceRefs: occurrenceRefs(30, 30)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "recalled", SourceRefs: occurrenceRefs(10, 10)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "recalled", SourceRefs: occurrenceRefs(20, 10)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "planned", SourceRefs: occurrenceRefs(10, 10)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "visited", SourceRefs: append(occurrenceRefs(10, 10), occurrenceRefs(10, 10)...)},
|
||||
{Name: first.Name, RegistryRefs: registryRefs(first), Kind: "visited", SourceRefs: occurrenceRefs(20, 20)},
|
||||
@@ -49,6 +49,9 @@ func TestExtractMapsKindsOrdersOccurrencesAndPreservesIndependentFacts(t *testin
|
||||
if !reflect.DeepEqual(got[0].SourceRefs, []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 10, EndUnitID: 10}}) {
|
||||
t.Fatalf("canonical evidence = %#v", got[0].SourceRefs)
|
||||
}
|
||||
if !reflect.DeepEqual(got[2].SourceRefs, []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 10, EndUnitID: 20}}) {
|
||||
t.Fatalf("reversed evidence = %#v", got[2].SourceRefs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractResolvesContextualSelectorsAndUsesCurrentTranscriptEvidenceOnly(t *testing.T) {
|
||||
|
||||
@@ -42,7 +42,7 @@ func canonicalizeLocation(location *locationResponse, order shared.SourceRefOrde
|
||||
if location == nil {
|
||||
return 0, false
|
||||
}
|
||||
refs := order.Canonicalize(canonicalSourceRefs(location.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(location.SourceRefs, sourceID)))
|
||||
location.SourceRefs = locationResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/location-registry"
|
||||
mappingPolicy = "dnd.location_registry.extract_mapping.v1"
|
||||
mappingPolicy = "dnd.location_registry.extract_mapping.v2"
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{"chunks", "source.transcript"}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func TestExtractMapsLocationsWithOwnedEvidenceAndDeterministicOrder(t *testing.T) {
|
||||
client := &fakeLocationsLLMClient{response: extractionResponse{Locations: []locationResponse{
|
||||
{Name: "The Tavern", SourceRefs: responseSourceRefs(3, 3)},
|
||||
{Name: "The Tavern", SourceRefs: responseSourceRefs(3, 2)},
|
||||
{Name: "Old Mill", SourceRefs: []locationSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 1}, {StartUnitID: 1, EndUnitID: 1}}},
|
||||
}}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
@@ -25,7 +25,7 @@ func TestExtractMapsLocationsWithOwnedEvidenceAndDeterministicOrder(t *testing.T
|
||||
refs := []source.SourceRef{{SourceID: "session-locations", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-locations", StartUnitID: 2, EndUnitID: 2}}
|
||||
want := dnd.LocationRegistry{Locations: []dnd.Location{
|
||||
{ID: identity.DeriveID("Old Mill", refs), Name: "Old Mill", SourceRefs: refs},
|
||||
{ID: identity.DeriveID("The Tavern", []source.SourceRef{{SourceID: "session-locations", StartUnitID: 3, EndUnitID: 3}}), Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: "session-locations", StartUnitID: 3, EndUnitID: 3}}},
|
||||
{ID: identity.DeriveID("The Tavern", []source.SourceRef{{SourceID: "session-locations", StartUnitID: 2, EndUnitID: 3}}), Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: "session-locations", StartUnitID: 2, EndUnitID: 3}}},
|
||||
}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
|
||||
@@ -47,7 +47,7 @@ func canonicalizeOccurrence(occurrence *occurrenceResponse, order shared.SourceR
|
||||
if occurrence == nil {
|
||||
return 0, false
|
||||
}
|
||||
refs := order.Canonicalize(canonicalSourceRefs(occurrence.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(occurrence.SourceRefs, sourceID)))
|
||||
occurrence.SourceRefs = occurrenceResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
const (
|
||||
Key = "dnd/npc-occurrences"
|
||||
mappingPolicy = "dnd.npc_occurrences.extract_mapping.v3"
|
||||
mappingPolicy = "dnd.npc_occurrences.extract_mapping.v4"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -22,9 +22,9 @@ func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
|
||||
{Name: "Opponent", Kind: "combat_opponent", SourceRefs: occurrenceRefs(20, 20)},
|
||||
{Name: "Ally", Kind: "combat_ally", SourceRefs: occurrenceRefs(5, 5)},
|
||||
{Name: "Speaker", Kind: "dialogue", SourceRefs: append(occurrenceRefs(2, 2), occurrenceRefs(2, 2)...)},
|
||||
{Name: "Present", Kind: "noncombat_presence", SourceRefs: occurrenceRefs(7, 7)},
|
||||
{Name: "Present", Kind: "noncombat_presence", SourceRefs: occurrenceRefs(7, 2)},
|
||||
{Name: "Mentioned", Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)},
|
||||
{Name: "Invalid", Kind: "unsupported", SourceRefs: occurrenceRefs(0, 0)},
|
||||
{Name: "Invalid", Kind: "other", SourceRefs: occurrenceRefs(0, 0)},
|
||||
}}}
|
||||
references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid")
|
||||
req := extractionRequest()
|
||||
@@ -44,17 +44,20 @@ func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
|
||||
dnd.NPCOccurrenceKindCombatAlly,
|
||||
dnd.NPCOccurrenceKindCombatOpponent,
|
||||
dnd.NPCOccurrenceKindOther,
|
||||
"unsupported",
|
||||
dnd.NPCOccurrenceKindOther,
|
||||
}) {
|
||||
t.Fatalf("occurrence kinds = %#v", got)
|
||||
}
|
||||
if refs := result.Value.Occurrences[1].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||
t.Fatalf("canonical source refs = %#v", refs)
|
||||
}
|
||||
if refs := result.Value.Occurrences[2].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 7}}) {
|
||||
t.Fatalf("reversed source refs = %#v", refs)
|
||||
}
|
||||
if id := result.Value.Occurrences[0].NPCID; id != identity.DeriveID("Mentioned") {
|
||||
t.Fatalf("durable NPC ID = %q, want registry identity", id)
|
||||
}
|
||||
if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != "unsupported" || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
|
||||
if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != dnd.NPCOccurrenceKindOther || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
|
||||
t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
|
||||
@@ -29,7 +31,6 @@ func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
|
||||
semanticCandidate := validOccurrenceResponse()
|
||||
occurrence := semanticCandidate["occurrences"].([]any)[0].(map[string]any)
|
||||
occurrence["name"] = ""
|
||||
occurrence["kind"] = "unsupported"
|
||||
ref := occurrence["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
@@ -38,12 +39,13 @@ func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("schema rejected validator-owned semantics: %v", err)
|
||||
t.Fatalf("schema rejected non-categorical validator-owned semantics: %v", err)
|
||||
}
|
||||
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(record map[string]any) { delete(record, "name") },
|
||||
func(record map[string]any) { record["kind"] = 1 },
|
||||
func(record map[string]any) { record["kind"] = "unsupported" },
|
||||
func(record map[string]any) { record["npc_id"] = "npc:sha256:opaque" },
|
||||
func(record map[string]any) { record["unexpected"] = true },
|
||||
func(record map[string]any) {
|
||||
@@ -79,6 +81,24 @@ func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.NPCOccurrenceKindMentioned), string(dnd.NPCOccurrenceKindNoncombatPresence), string(dnd.NPCOccurrenceKindDialogue), string(dnd.NPCOccurrenceKindCombatAlly), string(dnd.NPCOccurrenceKindCombatOpponent), string(dnd.NPCOccurrenceKindOther)} {
|
||||
candidate := validOccurrenceResponse()
|
||||
candidate["occurrences"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported NPC-occurrence kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validOccurrenceResponse() map[string]any {
|
||||
return map[string]any{"occurrences": []any{map[string]any{
|
||||
"name": "Mira Thorn", "kind": "dialogue",
|
||||
|
||||
@@ -46,7 +46,7 @@ func canonicalizeNPC(npc *npcResponse, order shared.SourceRefOrder, sourceID str
|
||||
if npc == nil {
|
||||
return 0, false
|
||||
}
|
||||
refs := order.Canonicalize(canonicalSourceRefs(npc.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(canonicalSourceRefs(npc.SourceRefs, sourceID)))
|
||||
npc.SourceRefs = npcResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
const Key = "dnd/npc-registry"
|
||||
|
||||
const mappingPolicy = "dnd.npc_registry.extract_mapping.v2"
|
||||
const mappingPolicy = "dnd.npc_registry.extract_mapping.v3"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{
|
||||
{
|
||||
Name: "Captain Vale", SourceRefs: responseSourceRefs(3, 3),
|
||||
Name: "Captain Vale", SourceRefs: responseSourceRefs(3, 2),
|
||||
},
|
||||
{
|
||||
Name: "Mira Thorn",
|
||||
@@ -34,7 +34,7 @@ func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) {
|
||||
}
|
||||
want := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}, {SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}},
|
||||
{ID: identity.DeriveID("Captain Vale"), Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 3, EndUnitID: 3}}},
|
||||
{ID: identity.DeriveID("Captain Vale"), Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 3}}},
|
||||
}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
|
||||
@@ -75,18 +75,18 @@ func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v", err)
|
||||
}
|
||||
if err := validateJSONSchema(t, map[string]any{"kind": "unrecognized", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
|
||||
if err := validateJSONSchema(t, map[string]any{"kind": "narrative", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("semantic candidate rejected by private schema: %v", err)
|
||||
}
|
||||
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
|
||||
Kind: dnd.SceneKind("unrecognized"), Title: " ", Summary: "",
|
||||
Kind: dnd.SceneKindNarrative, Title: " ", Summary: "",
|
||||
}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
scene := result.Value.Scenes[0]
|
||||
if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "" || scene.Summary != "" {
|
||||
if scene.Kind != dnd.SceneKindNarrative || scene.Title != "" || scene.Summary != "" {
|
||||
t.Fatalf("scene = %#v, want semantic candidates returned for deterministic validation", scene)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ var promptAssetManifest = shared.PromptAssetManifest{
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-scene-combat-policy.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,23 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||
}
|
||||
const combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
transcriptIndex := -1
|
||||
policyIndex := -1
|
||||
instructionsIndex := len(prepared.Messages) - 1
|
||||
policyOccurrences := 0
|
||||
for index, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, "scene-description-transcript") {
|
||||
transcriptIndex = index
|
||||
}
|
||||
if count := strings.Count(message.Content, combatPolicySentinel); count > 0 {
|
||||
policyIndex = index
|
||||
policyOccurrences += count
|
||||
}
|
||||
}
|
||||
if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex {
|
||||
t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) {
|
||||
@@ -30,7 +32,7 @@ func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.
|
||||
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
|
||||
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
|
||||
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
|
||||
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}, valid: true},
|
||||
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
|
||||
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
|
||||
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
|
||||
@@ -64,6 +66,19 @@ func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.SceneKindCombat), string(dnd.SceneKindNarrative), string(dnd.SceneKindRecap), string(dnd.SceneKindMeta)} {
|
||||
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
|
||||
if err := validateJSONSchema(t, response, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(instance)
|
||||
|
||||
@@ -45,7 +45,7 @@ func canonicalizeSpellCast(spell *spellCastResponse, order shared.SourceRefOrder
|
||||
if spell == nil {
|
||||
return 0, false
|
||||
}
|
||||
refs := order.Canonicalize(spellSourceRefs(spell.SourceRefs, sourceID))
|
||||
refs := order.Canonicalize(order.OrderEndpoints(spellSourceRefs(spell.SourceRefs, sourceID)))
|
||||
spell.SourceRefs = spellResponseRefs(refs)
|
||||
return order.EarliestValid(refs)
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
const Key = "dnd/spells"
|
||||
const SchemaVersion = "v1"
|
||||
|
||||
const mappingPolicy = "dnd.spells.extract_mapping.v2"
|
||||
const mappingPolicy = "dnd.spells.extract_mapping.v3"
|
||||
|
||||
const (
|
||||
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
||||
{
|
||||
Caster: " Aria ",
|
||||
Spell: " Cure Wounds ",
|
||||
SourceRefs: responseSourceRefs(1, 2),
|
||||
SourceRefs: responseSourceRefs(2, 1),
|
||||
},
|
||||
}}}
|
||||
req := extractionRequest()
|
||||
|
||||
@@ -23,7 +23,7 @@ const (
|
||||
Key = "dnd/item-registry"
|
||||
PromptID = "dnd.item_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.item_registry.normalize.v3"
|
||||
normalizationPolicy = "dnd.item_registry.normalize.v4"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
|
||||
ReasonCodeItemFieldsNormalized = "item_fields_normalized"
|
||||
@@ -129,7 +129,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1))
|
||||
return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
@@ -150,22 +150,38 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1))
|
||||
return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
}
|
||||
|
||||
func retryResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, reconciliation semanticreconcile.Result, rejectedGroups int) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
details := semanticreconcile.IssueDetails(reconciliation.Issues())
|
||||
issues := reconciliation.Issues()
|
||||
operatorDetails := semanticreconcile.IssueDetails(issues)
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
}
|
||||
var additionalGuidance []string
|
||||
if rejectedGroups > 0 {
|
||||
details = append(details, "currency may only be consolidated with aliases of one denomination")
|
||||
const currencyGuidance = "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items."
|
||||
operatorDetails = append(operatorDetails, "currency may only be consolidated with aliases of one denomination")
|
||||
correctionDetails = append(correctionDetails, currencyGuidance)
|
||||
additionalGuidance = append(additionalGuidance, currencyGuidance)
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance(additionalGuidance...)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
}
|
||||
discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
|
||||
return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, diagnostics.Aggregate("semantic proposal requires retry", details), semanticFallbackFinding(discardedGroups))
|
||||
return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", operatorDetails), semanticFallbackFinding(discardedGroups, correctionDetails...))
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discarded int) diagnostics.Finding {
|
||||
message := "semantic proposal could not be applied"
|
||||
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discarded >= 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discarded)
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
message = diagnostics.Aggregate(message, details)
|
||||
}
|
||||
return diagnostics.Finding{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
@@ -196,7 +212,7 @@ func fallbackResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnos
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate, reasonCode, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate, reasonCode, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, advisoryFindings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
@@ -205,7 +221,7 @@ func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, FallbackDiagnostics: fallbackGroups}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
||||
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
@@ -159,7 +159,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Piece", SourceRefs: ref(20)},
|
||||
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2,3],"canonical_candidate_number":2}]}`,
|
||||
wantNames: []string{"Gold Piece"},
|
||||
wantRefCounts: []int{3},
|
||||
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
||||
@@ -171,7 +171,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Silver Pieces", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -184,7 +184,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Longsword", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -197,7 +197,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Star Compass", SourceRefs: ref(10)},
|
||||
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
|
||||
wantNames: []string{"Compass of the Stars"},
|
||||
wantRefCounts: []int{2},
|
||||
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
||||
@@ -209,7 +209,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Longsword", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -253,7 +253,7 @@ func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
||||
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1},{"candidate_ids":[2,3],"canonical_candidate_id":3},{"candidate_ids":[1,3],"canonical_candidate_id":99}]}`}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,3],"canonical_candidate_number":99}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
||||
@@ -272,7 +272,7 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
{Name: "Silver Pieces", SourceRefs: ref(30)},
|
||||
{Name: "Rope", SourceRefs: ref(10)},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,4],"canonical_candidate_id":3},{"candidate_ids":[5,99],"canonical_candidate_id":5}]}`}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,4],"canonical_candidate_number":3},{"candidate_numbers":[5,99],"canonical_candidate_number":5}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err)
|
||||
@@ -289,7 +289,10 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
|
||||
t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") {
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "aliases of the same denomination") || !strings.Contains(result.Retry.CorrectionGuidance, "do not combine currency with non-currency items") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 5") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeItemSemanticRetryProposalInvalid) {
|
||||
t.Fatalf("correction guidance = %q, want shared and item-specific prose only", result.Retry.CorrectionGuidance)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "same denomination") {
|
||||
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
|
||||
}
|
||||
}
|
||||
@@ -321,6 +324,9 @@ func TestNormalizeRetryFallbackErrorsAndIdempotence(t *testing.T) {
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry with final-cause fallback", invalid.Retry)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
@@ -349,11 +355,11 @@ func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) {
|
||||
|
||||
@@ -24,7 +24,7 @@ const (
|
||||
Key = "dnd/location-registry"
|
||||
PromptID = "dnd.location_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.location_registry.normalize.v3"
|
||||
normalizationPolicy = "dnd.location_registry.normalize.v4"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
|
||||
ReasonCodeLocationFieldsNormalized = "location_fields_normalized"
|
||||
@@ -129,7 +129,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1))
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
@@ -149,17 +149,29 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1))
|
||||
return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
}
|
||||
|
||||
func retryResult(value dnd.LocationRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(reconciliation.Issues())), semanticFallbackFinding(reconciliation.DiscardedGroupCount()))
|
||||
issues := reconciliation.Issues()
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
}
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...))
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discarded int) diagnostics.Finding {
|
||||
message := "semantic proposal could not be applied"
|
||||
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discarded >= 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discarded)
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
message = diagnostics.Aggregate(message, details)
|
||||
}
|
||||
return diagnostics.Finding{Scope: "locations", ReasonCode: ReasonCodeLocationSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
@@ -185,7 +197,7 @@ func fallbackResult(value dnd.LocationRegistry, findings []diagnostics.Finding,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
func retryResultWithFallback(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
@@ -194,7 +206,7 @@ func retryResultWithFallback(value dnd.LocationRegistry, findings []diagnostics.
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: message, FallbackDiagnostics: fallbackGroups}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ func BenchmarkExactDuplicateGroupsManyDistinct(b *testing.B) {
|
||||
}
|
||||
|
||||
func TestNormalizeAppliesSafeAliasGroupAndUsesContextualInputs(t *testing.T) {
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`}
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
||||
doc := semanticDocument()
|
||||
input := dnd.LocationRegistry{Locations: []dnd.Location{
|
||||
{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
@@ -123,12 +123,15 @@ func TestNormalizeRejectsUnsafeAndOverlappingGroupsWithoutLosingCandidates(t *te
|
||||
{Name: "Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1},{"candidate_ids":[2,3],"canonical_candidate_id":3}]}`}
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Locations) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") {
|
||||
t.Fatalf("Normalize() = %#v, %v; want safe retry fallback", result, err)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group") {
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate groups 1, 2") || !strings.Contains(result.Retry.CorrectionGuidance, "at most one duplicate group") || strings.Contains(result.Retry.CorrectionGuidance, "overlapping_member") {
|
||||
t.Fatalf("correction guidance = %q, want model-facing overlap explanation", result.Retry.CorrectionGuidance)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("fallback diagnostics = %#v", result.Retry.FallbackDiagnostics)
|
||||
}
|
||||
}
|
||||
@@ -149,14 +152,14 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
}
|
||||
var candidateInput struct {
|
||||
Candidates []struct {
|
||||
CandidateID int `json:"candidate_id"`
|
||||
Label string `json:"label"`
|
||||
CandidateNumber int `json:"candidate_number"`
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateID != 1 || candidateInput.Candidates[1].CandidateID != 2 || candidateInput.Candidates[0].Label != "The Tavern" || candidateInput.Candidates[1].Label != "The Tavern" {
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Tavern" || candidateInput.Candidates[1].Label != "The Tavern" {
|
||||
t.Fatalf("candidate input = %#v, want distinct integer handles for equal names", candidateInput)
|
||||
}
|
||||
}
|
||||
@@ -188,6 +191,9 @@ func TestNormalizeHandlesRetryFallbackAndErrors(t *testing.T) {
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeLocationSemanticProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", invalid.Retry)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingLocationNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
|
||||
@@ -28,11 +28,11 @@ func TestRegisterPromptAssetsPreparesLocationNormalizationPrompt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "location-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"The Tavern","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "location-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"The Tavern","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") || !strings.Contains(prepared.Messages[2].Content, "same physical place") || !strings.Contains(prepared.Messages[2].Content, "parent and child places") {
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "same physical place") || !strings.Contains(prepared.Messages[2].Content, "parent and child places") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
for _, index := range []int{2, 4} {
|
||||
|
||||
@@ -23,7 +23,7 @@ const (
|
||||
Key = "dnd/npc-registry"
|
||||
PromptID = "dnd.npc_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.npc_registry.normalize.v5"
|
||||
normalizationPolicy = "dnd.npc_registry.normalize.v6"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
|
||||
ReasonCodeNPCFieldsNormalized = "npc_fields_normalized"
|
||||
@@ -128,7 +128,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1))
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
@@ -148,17 +148,29 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.NPCRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1))
|
||||
return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
}
|
||||
|
||||
func retryResult(value dnd.NPCRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(reconciliation.Issues())), semanticFallbackFinding(reconciliation.DiscardedGroupCount()))
|
||||
issues := reconciliation.Issues()
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
}
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...))
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discardedGroups int) diagnostics.Finding {
|
||||
message := "semantic proposal could not be applied"
|
||||
func semanticFallbackFinding(discardedGroups int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discardedGroups >= 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discardedGroups)
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discardedGroups)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
message = diagnostics.Aggregate(message, details)
|
||||
}
|
||||
return diagnostics.Finding{Scope: "npcs", ReasonCode: ReasonCodeNPCSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
@@ -184,7 +196,7 @@ func fallbackResult(value dnd.NPCRegistry, findings []diagnostics.Finding, fallb
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
func retryResultWithFallback(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
@@ -193,7 +205,7 @@ func retryResultWithFallback(value dnd.NPCRegistry, findings []diagnostics.Findi
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: message, FallbackDiagnostics: fallbackGroups}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "normalize-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Mira","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Mira","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
|
||||
},
|
||||
})
|
||||
@@ -49,10 +49,10 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
if prepared.Messages[0].Role != "system" {
|
||||
t.Fatalf("initial message role = %q, want system", prepared.Messages[0].Role)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") {
|
||||
if !strings.Contains(prepared.Messages[1].Content, "candidate_number") || !strings.Contains(prepared.Messages[1].Content, "integer") {
|
||||
t.Fatalf("protocol message = %q, want shared integer-handle protocol", prepared.Messages[1].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "same individual") || strings.Contains(prepared.Messages[2].Content, "source ranges") {
|
||||
if !strings.Contains(prepared.Messages[2].Content, "same individual") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || strings.Contains(prepared.Messages[2].Content, "source ranges") {
|
||||
t.Fatalf("NPC policy message = %q, want domain distinctions without copied ranges", prepared.Messages[2].Content)
|
||||
}
|
||||
for _, index := range []int{2, 4} {
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing
|
||||
}
|
||||
|
||||
func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`}
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
input := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
@@ -88,19 +88,19 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
}
|
||||
var visible struct {
|
||||
Candidates []struct {
|
||||
CandidateID int `json:"candidate_id"`
|
||||
CandidateNumber int `json:"candidate_number"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(completion.Inputs["candidates"].Content, &visible); err != nil {
|
||||
t.Fatalf("decode candidates: %v", err)
|
||||
}
|
||||
if got := []int{visible.Candidates[0].CandidateID, visible.Candidates[1].CandidateID, visible.Candidates[2].CandidateID}; !reflect.DeepEqual(got, []int{1, 2, 3}) {
|
||||
t.Fatalf("candidate IDs = %v, want contiguous request-local handles", got)
|
||||
if got := []int{visible.Candidates[0].CandidateNumber, visible.Candidates[1].CandidateNumber, visible.Candidates[2].CandidateNumber}; !reflect.DeepEqual(got, []int{1, 2, 3}) {
|
||||
t.Fatalf("candidate numbers = %v, want contiguous request-local handles", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,99],"canonical_candidate_id":3}]}`}
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
input := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
@@ -115,10 +115,13 @@ func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) {
|
||||
if result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || !strings.Contains(result.Retry.Message, "group 1: member_unknown") {
|
||||
t.Fatalf("retry = %#v, want bounded invalid-proposal diagnostics", result.Retry)
|
||||
}
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate group 2") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 3") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeNPCSemanticProposalInvalid) {
|
||||
t.Fatalf("correction guidance = %q, want contextual prose without internal labels", result.Retry.CorrectionGuidance)
|
||||
}
|
||||
if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Mira Thorn" || result.Value.NPCs[1].Name != "Captain Vale" {
|
||||
t.Fatalf("fallback NPCs = %#v, want independently safe group applied", result.Value.NPCs)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeNPCSemanticReconciliationExhausted || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "1 proposal group") {
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeNPCSemanticReconciliationExhausted || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "1 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "candidate-number range 1 through 3") {
|
||||
t.Fatalf("fallback diagnostics = %#v, want exact omitted-group warning", result.Retry.FallbackDiagnostics)
|
||||
}
|
||||
}
|
||||
@@ -134,6 +137,9 @@ func TestNormalizeInvalidStructuredOutputAndOperationalErrorsRemainDistinct(t *t
|
||||
if err != nil || result.Retry == nil || result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || len(result.Value.NPCs) != 2 {
|
||||
t.Fatalf("invalid structured result = %#v, %v; want deterministic retry fallback", result, err)
|
||||
}
|
||||
if result.Retry.CorrectionGuidance != "" || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", result.Retry)
|
||||
}
|
||||
operational := errors.New("provider unavailable")
|
||||
if _, err := newNormalizer(t, &recordingNPCNormalizerClient{err: operational}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)); !errors.Is(err, operational) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
t.Fatalf("operational completion error = %v, want ordinary error", err)
|
||||
@@ -178,8 +184,8 @@ func TestNormalizeRedactsContextMaterialFailures(t *testing.T) {
|
||||
|
||||
func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{responses: []string{
|
||||
`{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,99],"canonical_candidate_id":3}]}`,
|
||||
`{"duplicate_groups":[{"candidate_ids":[1,3],"canonical_candidate_id":3}]}`,
|
||||
`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`,
|
||||
`{"duplicate_groups":[{"candidate_numbers":[1,3],"canonical_candidate_number":3}]}`,
|
||||
}}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
@@ -202,7 +208,7 @@ func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNormalizeRetainsRecordsWhenAProposalUsesAnUnknownHandle(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,99],"canonical_candidate_id":1}]}`}
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
input := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
@@ -235,14 +241,14 @@ func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testi
|
||||
}
|
||||
var candidateInput struct {
|
||||
Candidates []struct {
|
||||
CandidateID int `json:"candidate_id"`
|
||||
Label string `json:"label"`
|
||||
CandidateNumber int `json:"candidate_number"`
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateID != 1 || candidateInput.Candidates[1].CandidateID != 2 || candidateInput.Candidates[0].Label != "The Guard" || candidateInput.Candidates[1].Label != "The Guard" {
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Guard" || candidateInput.Candidates[1].Label != "The Guard" {
|
||||
t.Fatalf("candidate input = %#v, want distinct integer handles for equal descriptors", candidateInput)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
)
|
||||
@@ -157,6 +158,7 @@ func registerPromptAssets(assets *llm.AssetRegistry) error {
|
||||
{name: "item registry normalization prompt assets", register: func() error { return itemregistrynormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "npc occurrences prompt assets", register: func() error { return occurrenceextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene descriptions prompt assets", register: func() error { return scenedescriptionextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene combat-semantics validator prompt assets", register: func() error { return combatsemantics.RegisterPromptAssets(assets) }},
|
||||
{name: "location registry prompt assets", register: func() error { return locationextract.RegisterPromptAssets(assets) }},
|
||||
{name: "location normalization prompt assets", register: func() error { return locationnormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "location occurrences prompt assets", register: func() error { return locationoccurrenceextract.RegisterPromptAssets(assets) }},
|
||||
|
||||
@@ -23,13 +23,15 @@ import (
|
||||
|
||||
func TestExtractionPromptComposition(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
endpointOrderSentinel = "according to transcript order"
|
||||
combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
)
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := registerPromptAssets(registry); err != nil {
|
||||
@@ -43,19 +45,20 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
middleSentinel string
|
||||
}{
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "item occurrences", promptID: itemoccurrenceextract.PromptID, promptVersion: itemoccurrenceextract.SchemaVersion, inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
||||
}), suffixGroups: [][]string{{evidenceSentinel}, {"item-registry-sentinel"}}},
|
||||
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
|
||||
}), suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"item-registry-sentinel"}}},
|
||||
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs, middleSentinel: combatPolicySentinel},
|
||||
{
|
||||
name: "combat turns",
|
||||
promptID: combatextract.PromptID,
|
||||
@@ -63,7 +66,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}},
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "enemy events",
|
||||
@@ -75,7 +78,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
"npc_occurrences": promptkit.Inline(`{"sentinel":"npc-occurrences-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{
|
||||
{evidenceSentinel},
|
||||
{evidenceSentinel, endpointOrderSentinel},
|
||||
{npcSentinel},
|
||||
{"combat-turns-sentinel", "npc-occurrences-sentinel"},
|
||||
},
|
||||
@@ -87,7 +90,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}},
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "location occurrences",
|
||||
@@ -96,7 +99,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"location_registry": promptkit.Inline(`{"sentinel":"location-registry-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {"location-registry-sentinel"}},
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"location-registry-sentinel"}},
|
||||
},
|
||||
{
|
||||
name: "spells",
|
||||
@@ -106,7 +109,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}, {catalogSentinel}},
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}, {catalogSentinel}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -155,6 +158,12 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
previousIndex = groupIndex
|
||||
}
|
||||
instructionIndex := len(prepared.Messages) - 1
|
||||
if testCase.middleSentinel != "" {
|
||||
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
|
||||
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
|
||||
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
|
||||
}
|
||||
}
|
||||
if instructionIndex <= previousIndex {
|
||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user