diff --git a/assets/dnd/item-registry/normalize/prompts/instructions.md b/assets/dnd/item-registry/normalize/prompts/instructions.md index 640e355..f719e46 100644 --- a/assets/dnd/item-registry/normalize/prompts/instructions.md +++ b/assets/dnd/item-registry/normalize/prompts/instructions.md @@ -1,9 +1,9 @@ -Use the supplied positive integer candidate IDs and their cited transcript -windows only to determine whether candidates identify the same item type or -unique designation. Do not treat nearby evidence, similar objects, or a shared -owner as sufficient. Keep currency denominations, materially different item -types, and uncertain aliases separate. Do not infer an item property or -uniqueness. +Determine whether candidates identify the same item type or unique designation +using their contextual labels and cited transcript windows. Do not treat nearby +evidence, similar objects, or a shared owner as sufficient. + +Keep currency denominations and materially different item types separate. Keep +uncertain aliases separate. Do not infer an item property or uniqueness. When selecting a canonical display name, choose one supplied candidate name that is the clearest established designation. diff --git a/assets/dnd/location-registry/normalize/prompts/instructions.md b/assets/dnd/location-registry/normalize/prompts/instructions.md index 7575e96..d24d908 100644 --- a/assets/dnd/location-registry/normalize/prompts/instructions.md +++ b/assets/dnd/location-registry/normalize/prompts/instructions.md @@ -1,7 +1,8 @@ -Use the supplied positive integer candidate IDs and their cited transcript -windows to determine whether candidates identify the same physical place. Do -not treat matching names, nearby evidence, nested places, or generic labels as -sufficient. Keep parent and child places, similarly named places, and uncertain -aliases separate. +Determine whether candidates identify the same physical place using their +contextual labels and cited transcript windows. Do not treat matching names, +nearby evidence, nested places, or generic labels as sufficient. + +Keep parent and child places separate, as well as similarly named places and +uncertain aliases. When selecting a canonical display name, prefer the clearest established name. diff --git a/assets/dnd/npc-registry/normalize/prompts/instructions.md b/assets/dnd/npc-registry/normalize/prompts/instructions.md index 0e2b3d6..1317b2c 100644 --- a/assets/dnd/npc-registry/normalize/prompts/instructions.md +++ b/assets/dnd/npc-registry/normalize/prompts/instructions.md @@ -1,7 +1,7 @@ -Use the supplied positive integer candidate IDs and their cited transcript -windows to determine whether candidates refer to the same individual. Preserve -distinct individuals even when their names are similar or their contextual -descriptions are identical. +Determine whether candidates refer to the same individual using their +contextual labels and cited transcript windows. Preserve distinct individuals +even when their names are similar or their contextual descriptions are +identical. When selecting a canonical display name, prefer a complete, stable proper name over an abbreviation. Prefer an unadorned proper name over that name plus a diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index 2fca1dc..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,809 +0,0 @@ -# Semantic Reconciliation Implementation Plan - -## Purpose - -This document is the ordered implementation plan for the target state defined -in the [Semantic Reconciliation Roadmap](semantic-reconciliation.md). Each -numbered stage is sized for one gpt-5.6-terra implementation prompt and must be -completed in order. - -The feature roadmap owns product intent and durable policy. This document owns -implementation sequence, package placement, private contracts, migration -mechanics, and verification. When a detail below conflicts with a current -implementation detail, preserve the feature-roadmap policy and deliberately -replace the superseded implementation. - -## Execution Rules - -For every stage: - -1. Read `docs/development.md`, all files under `docs/policy/`, the feature - roadmap, the files named by the stage, and focused tests before editing. -2. Implement only the assigned stage and prerequisites left incomplete by an - earlier stage. Do not begin a later migration opportunistically. -3. Preserve unrelated user changes and public module keys. Do not weaken typed - artifact registration or introduce untyped JSON mutation. -4. Keep all default tests deterministic, offline, and credential-free. Use a - small fake structured-LLM client only at the external completion boundary. -5. Test package-level behavior and consequential invariants. Do not add prompt - length, exact asset hash, private constant, or collaborator-choreography - change detectors. -6. Run `gofmt` on changed Go files, the focused commands listed for the stage, - and `git diff --check`. Resolve failures before handing off the stage. -7. Do not update current-behavior documentation before Stage 11. ADR-0013 is - the documentation-policy exception and may record the accepted decision - before the complete implementation exists. - -## Fixed Implementation Decisions - -The following decisions are not left to individual stages: - -- The shared package is `internal/framework/semanticreconcile`. It is - domain-neutral framework support, not a configured pipeline module and not a - D&D package. -- The initial core supports source-backed entity candidates only. It accepts - generic source references and source documents but no D&D types. -- The model-facing candidate object uses `candidate_id`, `label`, and - source-free `source_refs`. Eligible visible candidates receive contiguous - integer IDs beginning with `1`. -- The private response uses `duplicate_groups`, `candidate_ids`, and - `canonical_candidate_id`. It contains no names, labels, evidence ranges, - durable IDs, or replacement records. -- The generic response contract is version `v1`, with schema key - `semantic_reconciliation_llm`, schema ID - `notarius.generic.semantic_reconciliation.llm`, schema name - `notarius_semantic_reconciliation_llm_v1`, and registered filename - `semantic_reconciliation_llm.v1.json`. -- The complete generic default prompt ID is - `generic.semantic_reconciliation`, version `v1`. It has no default LLM - profile; callers must pass the resolved normalization profile. -- Core prompt protocol and schema are mandatory. A domain prompt may replace - only the generic semantic-policy message and must continue to select the - core-owned protocol and input presentation assets. -- Default source-context limits are radius `2`, at most `128` eligible - candidates, and at most `262144` total serialized bytes across candidate and - transcript input materials. Define these through one core-owned default - limits value. Test custom limits relationally rather than duplicating the - production literals throughout tests. -- Fewer than two eligible candidates skips the LLM without warning. Exceeding - a deterministic limit skips the LLM, preserves the deterministic - preprocessed artifact, and emits a bounded domain fallback warning without a - futile retry. -- Invalid structured output requests the normalizer's existing retry behavior. - A structurally valid proposal may retain independent safe groups while - discarding invalid groups; any discarded group requests a retry, and retry - exhaustion uses the existing deterministic fallback-warning contract. -- Transport, provider, cancellation, and context-material construction errors - remain execution errors. They are not converted into empty semantic results. -- The shared application helper never emits domain warnings or derives durable - IDs. It owns partition application, stable ordering, preservation of - ungrouped records, and provenance bookkeeping; typed domain policy owns value - consolidation, group guards, IDs, and warning presentation. -- Production registration of the generic prompt and schema belongs to the - generic registrar, which already runs before the D&D registrar. D&D prompt - definitions may mount core-owned shared assets but remain owned and hashed by - their D&D normalizer packages. -- Existing D&D prompt IDs and prompt versions remain unchanged. The new generic - schema has its own `v1` identity, so no private selector-schema compatibility - layer is required. -- Bump normalization policies to `dnd.npc_registry.normalize.v5`, - `dnd.item_registry.normalize.v3`, and - `dnd.location_registry.normalize.v3` during their respective migrations. -- No public configuration, durable D&D schema, integration contract, module - key, or validator-chain change is part of this work. - -## Stage 1: Record The Semantic Reconciliation Decision - -### Goal - -Record the accepted architectural decision before introducing the reusable -mechanism, without describing unimplemented behavior in current architecture -or internal documentation. - -### Work - -- Create - `docs/adr/0013-use-request-local-candidate-handles-for-semantic-reconciliation.md` - using the repository's Nygard ADR format and `Status: Accepted`. -- Cite ADR-0003, ADR-0004, ADR-0009, and ADR-0012 where their typed-artifact, - package-boundary, evidence, and opaque-ID decisions apply. -- Record these decisions: - - semantic reconciliation is a framework mechanism used by typed normalize - stage modules; - - a model receives contextual candidate data but returns request-local - one-based integer handles only; - - the LLM proposes duplicate groups and a supplied canonical member; - - deterministic code validates and applies the proposal; - - a mandatory shared protocol is combined with an explicit generic or - domain-owned semantic policy; and - - durable IDs, copied contextual selectors, synthesized replacements, - reflection-based arbitrary JSON, and hidden cross-stage behavior are - rejected alternatives. -- State explicitly that acceptance does not imply implementation completion - and link to the feature and implementation roadmaps for status. -- Do not modify the accepted decision text of ADR-0012. - -### Acceptance Criteria - -- The ADR contains context, decision, alternatives, and consequences and does - not claim that the new core already exists. -- No current-behavior document outside `docs/adr/` or `docs/roadmap/` changes. -- All ADR and roadmap links resolve. - -### Validation - -```sh -git diff --check -``` - -## Stage 2: Build Bounded Candidate And Source-Context Preparation - -### Goal - -Create the domain-neutral candidate preparation boundary with contiguous -request-local IDs and bounded prompt materials, without changing any D&D -normalizer yet. - -### Work - -- Create `internal/framework/semanticreconcile` with package documentation and - source-context preparation code adapted from - `internal/modules/dnd/shared/entityreconcile/context.go`. Leave the old D&D - package in place until all migrations finish. -- Define a domain-neutral `Candidate` containing a contextual `Label` and owned - generic `[]source.SourceRef`. Do not add a durable ID field. -- Define `Limits` and one `DefaultLimits()` value with the fixed radius, - candidate, and serialized-material bounds above. Validation rejects negative - radius and non-positive maximums before doing work. -- Prepare candidates in caller order: - - validate every source reference against `source.DocumentIndex`; - - exclude a candidate with no references or any invalid reference; - - canonicalize its source-free ranges in source order and remove exact - duplicate ranges; - - assign IDs only to eligible candidates and make them contiguous from `1`; - and - - retain an internal ID-to-original-candidate-position mapping. -- Do not exclude two candidates merely because their labels and ranges are - identical. Their local integer IDs now disambiguate them. -- Serialize candidate input as an object containing `candidates`, where each - entry has `candidate_id`, `label`, and `source_refs` with only - `start_unit_id` and `end_unit_id`. -- Adapt the existing coalesced transcript-window algorithm. Preserve source - order, cloned unit metadata, and the `cited` marker; do not mutate the source - document or candidate references. -- Return an explicit preparation disposition for: - - ready materials; - - fewer than two eligible candidates; and - - candidate or combined-material limit exceeded. -- Enforce the candidate limit before rendering large materials and the byte - limit over the sum of serialized candidate and transcript material. Do not - split the request. -- Construct `contracts.LLMInputMaterial` values with stable content type, - content digest, and the existing `candidates` and `transcript` input names. -- Expose owned accessors for visible candidate mappings and materials. Do not - expose mutable retained slices or maps. - -### Tests - -- Port and revise the valuable behavior tests from `entityreconcile`, replacing - selector-collision expectations with contiguous-ID behavior. -- Cover invalid and missing references, non-sequential source unit IDs, - coalesced windows, candidate filtering, identical descriptors, nil source, - input ownership, and deterministic serialization. -- Use custom small limits to prove exactly-at-limit acceptance and one-over - skipping for candidate count and material bytes. Do not assert the production - default literals merely as change detectors. -- Prove model material contains no source document ID or application entity ID. - -### Acceptance Criteria - -- The package imports no D&D or production-module package. -- Same-label and same-range candidates remain independently selectable. -- No LLM client or response schema is introduced in this stage. -- Existing D&D behavior remains unchanged and all prior tests still pass. - -### Validation - -```sh -go test ./internal/framework/semanticreconcile -go test ./internal/modules/dnd/shared/entityreconcile -git diff --check -``` - -## Stage 3: Implement The Integer Proposal Contract And Assessor - -### Goal - -Add the private integer response schema and a pure deterministic assessor that -turns model proposals into immutable, safely ordered reconciliation plans. - -### Work - -- Add core response types corresponding exactly to: - - `duplicate_groups`; - - each group's `candidate_ids`; and - - `canonical_candidate_id`. -- Add - `assets/generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json` - with Draft 2020-12 metadata and the fixed schema identity above. -- Require all fields, reject unknown fields, require at least two group members, - require positive integers, and use `uniqueItems: true` for member IDs. Keep - application semantic validation authoritative. -- Add schema loading metadata and helpers to the core package, following the - existing `internal/framework/llm` response-schema pattern. -- Implement assessment against the prepared request mapping: - - reject zero, negative, or unknown member and canonical IDs; - - reject repeated members even if schema validation was bypassed in a unit - test; - - reject groups with fewer than two distinct valid members; - - reject a canonical ID that is not a valid member; - - mark every otherwise-local-valid group sharing a member with another such - group as conflicting and discard all conflicting groups; - - retain independent safe groups even when another group is discarded; and - - report stable issue categories and original response group indexes for - retry diagnostics. -- Produce an immutable plan expressed in original candidate positions rather - than model-visible IDs. Sort members by original position and safe groups by - earliest member so response ordering cannot alter deterministic output. -- Provide owned accessors for plan groups, issues, discarded-group count, and - whether retry is required. - -### Tests - -- Cover empty proposals, valid multi-member groups, unknown IDs, repeated IDs, - canonical-not-member, too-small groups, overlapping groups, independent safe - plus invalid groups, and response-order independence. -- Compile and validate representative accepted and rejected JSON against the - actual schema offline. -- Prove returned plans and issues cannot mutate retained assessment state. -- Add a focused fuzz test only if it remains small and protects the partition - and no-panic invariants more efficiently than table cases. - -### Acceptance Criteria - -- The model response types contain no contextual selectors or source ranges. -- Invalid groups cannot enter the plan, and overlapping groups cannot be - partially accepted. -- Plan ordering depends only on deterministic candidate order. -- The old D&D response schema remains temporarily available for unmigrated - normalizers. - -### Validation - -```sh -go test ./internal/framework/semanticreconcile -git diff --check -``` - -## Stage 4: Add Generic Prompt Assets And Production Asset Registration - -### Goal - -Provide one complete generic default prompt, reusable mandatory protocol/input -assets for domain prompts, and one production registration owner for the new -prompt and schema. - -### Work - -- Replace the two draft files currently under - `assets/generic/normalize/deduplication/` with a coherent asset set: - - `prompts/prompt.yaml` for `generic.semantic_reconciliation` `v1`; - - `prompts/system.md` with provider-neutral structured reconciliation role; - - `prompts/protocol.md` with the mandatory integer selection and safety - contract; - - `prompts/instructions.md` with the conservative generic semantic default; - - `prompts/candidates.md`; and - - `prompts/transcript-windows.md`. -- The default prompt declares required `candidates` and `transcript` - `application/json` inputs, omits `default_profile`, selects the new schema, - uses zero PromptKit repair attempts, and orders messages from stable to - variable: - 1. generic system; - 2. mandatory protocol; - 3. generic semantic instructions, carrying the ephemeral cache marker for - the stable prefix; - 4. candidate material; and - 5. transcript windows. -- Keep protocol, semantic policy, and presentation as separate assets even - though the rendered prompt is compact. Domain prompts must be able to reuse - protocol and presentation while substituting only semantic instructions. -- In `internal/framework/semanticreconcile`, add scoped asset loading, - deterministic prompt/schema hashes, `RegisterAssets`, and a narrow allowlist - helper that returns core-owned shared prompt files for a consuming manifest. - The root `assets` package remains unchanged and contains no logic. -- Extend `internal/modules/dnd/shared.PromptAssetManifest` to accept explicitly - supplied external shared files in addition to named D&D shared files. Include - those external files in both the mounted prompt filesystem and prompt hash. - Reuse `promptfs.SharedPromptFile`; do not teach D&D shared code paths about a - hard-coded generic directory. -- Update `internal/modules/generic/register.Register` to require a non-nil asset - registry and register the semantic-reconciliation assets before its existing - modules and validators. Preserve contextual duplicate-registration errors. -- Do not yet change the three D&D normalization prompts or remove their old - schema registration. - -### Tests - -- Test the complete generic prompt and schema can be registered and prepared - offline with an explicit test profile and representative integer candidate - material. -- Verify the prepared output contract selects the generic schema and that the - rendered prompt asks for integer IDs rather than names or source ranges in - the response. Do not snapshot all prose or its exact length. -- Extend D&D manifest tests for external shared-file mounting, hashing, missing - files, and duplicate destinations at the stable manifest boundary. -- Update generic registrar tests for nil assets, successful asset registration, - and duplicate registration without duplicating PromptKit internals. - -### Acceptance Criteria - -- Generic production registration makes the complete default prompt and schema - available before D&D registration. -- Domain prompt manifests can mount the exact core protocol bytes without - copying them. -- The generic prompt has a stable prefix followed only by variable candidates - and transcript material. -- Existing D&D prompt behavior remains operational during the migration. - -### Validation - -```sh -go test ./internal/framework/semanticreconcile ./internal/framework/promptfs -go test ./internal/modules/generic/register ./internal/modules/dnd/shared -go test ./internal/cli -run 'Production|Catalog|Prompt' -git diff --check -``` - -## Stage 5: Add The Shared Structured-LLM Reconciliation Engine - -### Goal - -Centralize prompt execution and outcome classification behind one -provider-neutral engine while leaving typed artifact mutation to later stages. - -### Work - -- Add an `Engine` in `internal/framework/semanticreconcile` constructed with a - non-nil `contracts.StructuredLLMClient`, validated limits, and a prompt spec. - A prompt spec contains a non-empty prompt ID, version, and SHA-256 digest; it - may identify the generic default or a domain-owned prompt that obeys the same - schema. Validate the digest at construction so manifest metadata and - checkpoint fingerprints never silently omit the selected prompt bytes. -- Define a request containing stage name, source document, candidates, resolved - profile ID, and session ID. Preserve profile and session values exactly in - `contracts.StructuredCompletionRequest`. -- The engine must: - - reject nil engine, client, context, and invalid construction state with - contextual errors; - - honor context cancellation before material preparation and completion; - - use Stage 2 preparation and skip without calling the LLM for insufficient - or over-limit inputs; - - call `CompleteStructured` exactly once for a ready request, using only the - `candidates` and `transcript` materials; - - classify `contracts.ErrInvalidStructuredOutput` as a retryable semantic - outcome rather than a transport error; - - return other completion failures as errors with stage context; - - assess a decoded response through Stage 3; and - - return a safe plan, stable issues, discarded count, and a disposition that - distinguishes complete, retryable invalid structured output, retryable - discarded proposal groups, insufficient candidates, and deterministic - limit skip. -- Keep normalize-stage reason codes, warning text, and `NormalizeRetry` - construction out of the engine. The engine exposes neutral classifications - that typed normalizers translate through their existing policies. -- Expose core manifest metadata and checkpoint fingerprints for prompt ID, - version and hash, schema identity and hash, core reconciliation policy, and - complete limits. Domain normalizers append their identity and normalization - policy fingerprints rather than rebuilding core metadata independently. -- Fingerprint the limit policy as one canonical core-owned value so any limit - change invalidates relevant normalization checkpoints without test suites - duplicating every literal. - -### Tests - -- Use a compact recording fake client to cover request propagation, successful - assessment, empty groups, invalid structured output, transport failure, - cancellation, insufficient candidates, candidate limit, and material limit. -- Prove no LLM call occurs for deterministic skip outcomes. -- Prove repeated calls do not retain plans, issues, candidate mappings, or - warnings from earlier calls. -- Test metadata and fingerprints relationally: required categories must be - present and change when supplied prompt/schema/limit policy changes; do not - freeze current digest values. - -### Acceptance Criteria - -- Consuming modules no longer need to construct the common structured - completion request once migrated. -- The engine has no dependency on pipeline retry orchestration or a D&D type. -- Provider and transport behavior remain behind `StructuredLLMClient`. -- The old normalizers remain unchanged and compiling. - -### Validation - -```sh -go test ./internal/framework/semanticreconcile -go test ./internal/framework/llm ./internal/framework/contracts -git diff --check -``` - -## Stage 6: Add Generic Typed Plan Application - -### Goal - -Centralize safe partition application, ordering, and provenance bookkeeping for -typed records without moving domain consolidation rules into the core. - -### Work - -- Add a generic record envelope that owns a typed value, sorted unique original - input indexes, and earliest deterministic input position. Constructors and - accessors must preserve caller ownership. -- Add a deliberately small typed application policy with these responsibilities: - - clone one typed value; - - optionally reject an otherwise safe semantic group through a stable neutral - category; and - - consolidate owned member values using one supplied canonical member. -- Implement plan application that: - - validates plan positions against the provided record slice; - - never mutates records or policy inputs; - - preserves every ungrouped record as an owned clone; - - emits exactly one record for an accepted group; - - preserves all members unchanged when the typed guard rejects a group; - - unions and sorts original input indexes in core bookkeeping; - - sets the resulting earliest position to the earliest member; - - places output by earliest contributing position; and - - returns neutral applied-group and rejected-group events containing the - provenance needed for domain warning and retry construction. -- Treat a policy consolidation error as an execution error unless it is the - explicit safe-group rejection result. Do not silently preserve records after - an unexpected application failure. -- Do not make the helper aware of source references, names, durable IDs, - currencies, people, or locations. Domain consolidators continue to union and - canonicalize evidence inside their typed value and derive the final ID. - -### Tests - -- Use a minimal synthetic typed record to cover no groups, one and multiple - groups, ungrouped preservation, canonical member selection, output ordering, - provenance union, guard rejection, malformed plan positions, consolidation - failure, nil versus empty ownership, and non-mutation. -- Prefer table-driven invariant tests over tests for private loops or callback - invocation order. - -### Acceptance Criteria - -- A future domain adapter can apply a plan without reimplementing partition - traversal or provenance ordering. -- Rejected groups preserve all members exactly once. -- The helper cannot derive a domain ID or emit a domain warning. - -### Validation - -```sh -go test ./internal/framework/semanticreconcile -git diff --check -``` - -## Stage 7: Migrate NPC Registry Normalization - -### Goal - -Use the shared engine and typed application path for NPC reconciliation while -preserving the existing NPC artifact and failure semantics. - -### Work - -- Update the NPC normalization prompt manifest and assets: - - keep prompt ID `dnd.npc_registry.normalize` and version `v1`; - - retain the D&D system prompt and NPC-owned semantic instructions; - - mount the generic protocol, candidate presentation, and transcript-window - assets from `semanticreconcile`; - - order messages as D&D system, generic protocol, NPC semantic instructions - with the stable-prefix cache marker, candidates, then transcript windows; - and - - select `semantic_reconciliation_llm.v1.json`. -- Remove requirements for the model to copy names or ranges. Adjust NPC policy - prose to describe integer candidate selection while retaining all current - individual-person and canonical-name distinctions. -- Construct a shared engine in `npcregistry.New`, using the NPC prompt spec and - default limits. Replace direct `BuildContext`, `CompleteStructured`, and - `Assessment` orchestration with one engine call. -- Convert deterministic preprocessed NPC records into core typed envelopes and - candidates. Keep exact deterministic name/evidence normalization before the - semantic pass. -- Implement the narrow NPC typed policy: - - no additional group guard; - - choose the canonical member's normalized NPC name; - - union and source-order all member references; - - derive the final NPC ID through `npcs/identity`; and - - return an owned value. -- Translate engine and application outcomes to existing NPC warnings and retry - behavior. Safe independent groups may appear in the retry candidate result; - invalid structured output retries from the deterministic value; limit skip - returns the deterministic value with one bounded reconciliation-exhausted - warning and no retry. -- Preserve existing NPC reason-code strings and warning cap. Bump only the - normalization policy to `v5`. -- Replace locally assembled common prompt/schema/context metadata and - fingerprints with the core-provided values plus NPC identity and - normalization policy. Retain user-useful manifest field names where practical - but ensure the new generic schema identity and complete limits are recorded. -- Delete NPC-only key translation, safe-group traversal, and duplicated generic - application helpers made obsolete by the core. Retain NPC-specific - preprocessing, consolidation policy, and diagnostics. - -### Tests - -- Rewrite fake LLM responses to return integer IDs and inspect candidate input - only where needed to prove contiguous IDs and no durable identity leakage. -- Preserve behavior coverage for exact deterministic consolidation, semantic - aliases, canonical name choice, evidence union, ID recomputation, warnings, - invalid proposal retry, exhaustion fallback, cancellation, ownership, and - no-call cases. -- Add focused coverage for identical contextual NPC descriptors remaining - independently addressable and for a limit skip producing no LLM call. -- Update prompt preparation tests for the generic schema and shared protocol - without snapshotting full prompt prose or exact message length. - -### Acceptance Criteria - -- NPC normalization imports `semanticreconcile` and no longer imports the old - D&D `entityreconcile` package. -- Its model response requires only integer candidate handles. -- Durable NPC output and module key remain unchanged. -- Focused tests demonstrate behavior parity plus the safer selection protocol. - -### Validation - -```sh -go test ./internal/modules/dnd/normalize/npcregistry -go test ./internal/framework/semanticreconcile -go test ./internal/modules/dnd/register -run 'NPC|Prompt|Schema' -git diff --check -``` - -## Stage 8: Migrate Item Registry Normalization - -### Goal - -Move item registry reconciliation onto the shared path while preserving its -item-type and currency safety rules. - -### Work - -- Update the item normalization prompt exactly as in Stage 7, retaining prompt - ID `dnd.item_registry.normalize`, version `v1`, and item-owned semantic policy - while selecting the generic protocol, presentation assets, and schema. -- Construct and use the shared engine with default limits. Convert deterministic - item records into core candidates and typed envelopes. -- Implement the item typed policy: - - retain the existing currency-denomination group guard; - - reject a group mixing currency with non-currency or different - denominations through the core's explicit typed-group rejection path; - - choose the canonical member's normalized item name; - - union and source-order evidence; - - derive the item ID through `items/identity`; and - - preserve all current item-type and unique-designation semantics. -- Count a typed currency guard rejection with discarded model groups for retry - and exhaustion diagnostics. Preserve rejected group members unchanged and - exactly once. -- Translate insufficient, limit, invalid structured, invalid proposal, valid, - and transport outcomes consistently with NPC behavior while retaining item - reason codes and warning cap. -- Bump the item normalization policy to `v3`, adopt core metadata and - fingerprints, and remove item copies of generic key translation, group - traversal, and application logic. - -### Tests - -- Convert semantic response fixtures to integer IDs. -- Retain coverage for item aliases, canonical names, evidence, IDs, warnings, - retries, fallback, ownership, and no-call cases. -- Specifically protect same-denomination alias acceptance, cross-denomination - rejection, currency/non-currency rejection, preservation of every rejected - member, and combination of an accepted independent group with a rejected - group. -- Update prompt/schema preparation tests without exact prompt snapshots. - -### Acceptance Criteria - -- Item normalization no longer imports old `entityreconcile` code. -- Its domain guard is expressed only through the typed policy and cannot be - bypassed by a schema-valid proposal. -- Durable item output and public module behavior remain unchanged. - -### Validation - -```sh -go test ./internal/modules/dnd/normalize/itemregistry -go test ./internal/framework/semanticreconcile -go test ./internal/modules/dnd/register -run 'Item|Prompt|Schema' -git diff --check -``` - -## Stage 9: Migrate Location Registry Normalization - -### Goal - -Complete adoption by moving location registry reconciliation onto the shared -engine and typed application path. - -### Work - -- Update the location normalization prompt as in Stages 7 and 8, retaining - prompt ID `dnd.location_registry.normalize`, version `v1`, and the - location-owned physical-place semantic policy. -- Construct and use the shared engine with default limits. Convert deterministic - location records into core candidates and typed envelopes. -- Implement the location typed policy: - - no additional group guard beyond existing location semantic policy; - - choose the supplied canonical member's normalized location name; - - union and source-order all member references; - - derive the location ID from the final name and final references through - `locations/identity`; and - - preserve same-name, parent/child, and physical-place distinctions. -- Translate outcomes through existing location retry, fallback, reason-code, - and warning-cap behavior. Bump the normalization policy to `v3`. -- Adopt core metadata and fingerprints and delete location copies of generic - reconciliation and application mechanics. - -### Tests - -- Convert semantic response fixtures to integer IDs while retaining physical - place, same-name, canonical-name, evidence-dependent ID, warning, retry, - fallback, ownership, and no-call coverage. -- Prove same-name records with distinct evidence can both be addressed by IDs - without copied selectors. -- Update prompt/schema preparation tests without freezing prompt prose. - -### Acceptance Criteria - -- All three semantic D&D registry normalizers use the shared core. -- Location durable IDs are still derived only after final evidence union. -- No location code imports old `entityreconcile`. - -### Validation - -```sh -go test ./internal/modules/dnd/normalize/locationregistry -go test ./internal/framework/semanticreconcile -go test ./internal/modules/dnd/register -run 'Location|Prompt|Schema' -git diff --check -``` - -## Stage 10: Remove The Legacy Reconciliation Path And Consolidate Registration - -### Goal - -Delete superseded selector-copying code and assets, leave one schema and core, -and verify production composition as a whole. - -### Work - -- Delete `internal/modules/dnd/shared/entityreconcile` after confirming no - imports remain. -- Delete `assets/dnd/entity-reconciliation/` and - `assets/dnd/shared/prompts/common-dnd-entity-reconciliation.md`. -- Remove the obsolete shared prompt-name entry and the D&D registrar's old - entity-reconciliation schema registration. -- Remove any obsolete local candidate or transcript template files from the - three normalization asset trees after their manifests use the shared generic - presentation assets. Keep only their `prompt.yaml` and domain semantic - `instructions.md` unless another file remains genuinely domain-specific. -- Ensure the generic registrar is the sole production owner of generic - reconciliation prompt/schema registration and that the existing production - registrar order remains generic, Seriatim, D&D. -- Update registrar and CLI composition tests so they prove assembled production - assets are complete. A D&D registrar unit test may pre-register required - generic assets or limit itself to D&D-owned assets; do not duplicate generic - asset ownership inside D&D merely to preserve an old isolated-test setup. -- Search all Go, asset, and roadmap-adjacent current documentation for old - schema keys, selector response types, `common-dnd-entity-reconciliation.md`, - and model-output source ranges. Remove only obsolete uses; source ranges - remain valid model input. -- Review the three normalizers side by side. Consolidate any remaining - demonstrated neutral retry/adaptation helper into the core if it can be done - without domain warning semantics; otherwise keep the difference explicit. - -### Tests - -- Run the complete framework, generic module, D&D module, D&D registrar, and - production composition suites. -- Add or retain one assembled prompt/schema test per meaningful owner rather - than reproducing every core schema case in all three domains. -- Confirm duplicate production registration still fails contextually and no - prompt or schema path collides. - -### Acceptance Criteria - -- Exactly one semantic reconciliation response schema and assessor exist. -- No model-facing reconciliation response requires a name or evidence range. -- Production composition registers every selected D&D normalization prompt and - the generic schema exactly once. -- No compatibility shim or dead legacy asset remains. - -### Validation - -```sh -go test ./internal/framework/... -go test ./internal/modules/generic/... -go test ./internal/modules/dnd/... -go test ./internal/cli -run 'Production|Catalog|Prompt|Schema' -rg -n 'dnd_entity_reconcile_llm|common-dnd-entity-reconciliation|entityreconcile' internal assets docs/internal docs/policy docs/config.md -git diff --check -``` - -The `rg` command should return no live legacy references. Separately review -roadmap and ADR historical references rather than deleting accurate decision -history. - -## Stage 11: Finalize Current Documentation And Repository Verification - -### Goal - -Make current documentation accurately describe the implemented architecture, -then perform the complete offline verification pass. - -### Work - -- Update `docs/policy/architecture.md` to: - - distinguish an artifact family from one configured stage module without - changing the fixed pipeline; - - identify semantic reconciliation as a domain-neutral framework mechanism; - - state that request-local candidate handles are an approved application of - ADR-0012 and link ADR-0013; and - - retain typed domain mutation, evidence, and dependency-direction - invariants. -- Update `docs/internal/overview.md` with the implemented - `internal/framework/semanticreconcile` responsibility, linking to the focused - internal owners rather than duplicating mechanics. -- Update `docs/internal/modules.md` to document artifact-family ownership, - stage-module registration, and how a typed normalizer instantiates the shared - strategy. -- Update `docs/internal/dnd.md` to replace selector-copying descriptions with - the integer candidate protocol, generic protocol/domain policy composition, - limit behavior, typed application boundary, and the three registry-specific - rules. Keep exact public keys and validator chains in `docs/config.md` rather - than duplicating them. -- Update `docs/internal/llm.md` only as needed to identify ownership and - registration of the generic prompt/schema assets and checkpoint metadata. -- Do not change integration contracts unless repository inspection finds an - incorrect statement: the durable artifact shapes are intentionally - unchanged. Do not add user configuration documentation because this feature - adds no configuration. -- Confirm ADR-0013 accurately matches the final implementation. Do not rewrite - its accepted decision to accommodate accidental implementation drift; fix - the implementation or create a superseding decision if a genuine conflict - was discovered. -- Review `docs/roadmap/future.md` only for links and scope boundaries. Keep - batching, operator-selected policy, broader inputs, a universal module key, - and physical package reorganization deferred. -- Run formatting, tests, vet, build, whitespace, link, and stale-reference - checks. No command may contact a live LLM provider. - -### Acceptance Criteria - -- Current docs describe only implemented behavior and assign each fact to its - canonical owner. -- The architecture clearly separates artifact families, stage modules, shared - model judgment, and typed deterministic application. -- All durable D&D schemas, module keys, and examples remain valid. -- No stale selector-copying documentation or asset path remains. -- The repository-wide test, vet, and build checks pass offline. - -### Validation - -```sh -go test ./... -go vet ./... -go build ./cmd/notarius -git diff --check -``` - -Run `gofmt` on every changed Go file before these commands; do not pass package -directories to `gofmt`. - -Also verify every changed Markdown link target manually or with the repository's -available link checker. Review `git status --short` and the complete diff to -confirm that no unrelated file or generated secret-bearing material was added. diff --git a/docs/roadmap/semantic-reconciliation.md b/docs/roadmap/semantic-reconciliation.md deleted file mode 100644 index ad7cf58..0000000 --- a/docs/roadmap/semantic-reconciliation.md +++ /dev/null @@ -1,426 +0,0 @@ -# Semantic Reconciliation Roadmap - -## Purpose - -This roadmap defines a reusable, LLM-assisted semantic-reconciliation facility -for source-backed entity registries. The facility will centralize the common -candidate preparation, prompt execution, proposal validation, safety, retry, -and consolidation mechanics currently implemented by the D&D NPC, item, and -location registry normalizers while preserving typed, domain-owned output. - -The model will remain a constrained proposal source. Notarius will retain -authority over candidate identity, proposal validation, deterministic mutation, -provenance, durable identifiers, warnings, and final artifact construction. - -## Motivation - -NPC, item, and location registry normalization now demonstrate the same useful -pattern: deterministic preprocessing produces candidate records, an LLM judges -whether some candidates describe the same underlying entity, and deterministic -code applies only safe proposed groups. The current D&D-shared implementation -proves the approach, but it still duplicates orchestration and application -logic across normalizers and requires the model to reproduce complete -contextual selectors containing names and evidence ranges. - -The target design should provide one efficient and thoroughly tested -reconciliation core that can support additional artifact families without -moving domain semantics into generic code. It should also simplify the model's -task by replacing selector reproduction with small request-local integer -handles. - -## Goals - -- Establish one domain-neutral semantic-reconciliation core for source-backed - entity candidates. -- Move all demonstrated common mechanics into that core, including structured - LLM execution and deterministic proposal assessment. -- Preserve exact typed artifact ownership from merged input through normalized - output. -- Give the model semantic evidence while asking it to return only small, - request-local candidate identifiers. -- Provide a conservative generic semantic prompt policy that a typed artifact - family may use by default. -- Allow an artifact family to supply narrower domain semantic policy without - replacing the mandatory shared protocol and safety instructions. -- Have the NPC, item, and location registry normalizers use the shared core - without changing their durable artifact contracts, configured module keys, - domain identity rules, or fallback guarantees. -- Make addition of another eligible registry normalizer primarily an adapter - and policy exercise rather than a copy of reconciliation machinery. - -## Terminology And Ownership - -A **stage module** remains one configured implementation of one pipeline stage. -An extractor and a normalizer are separate stage modules even when they -collaborate on the same artifact kind. - -An **artifact family** is the cohesive domain feature that owns an artifact's -types, codec, extractor, merge choice, normalizer, validators, prompt policy, -schemas, identity helpers, and reference projections. For example, D&D spells -and the D&D NPC registry are artifact families whose implementations span -multiple explicit pipeline stages. - -This terminology clarifies existing ownership without changing Notarius's -fixed pipeline or combining stages. It is consistent with the accepted -domain-first organization and typed generic-strategy boundary in -[ADR-0004](../adr/0004-package-modules-by-domain.md). The D&D registrar remains -responsible for composing D&D-owned stage modules and their policies; the -generic core remains unaware of D&D types or semantics. - -## Target Architecture - -### Shared Core - -A domain-neutral framework package should own semantic reconciliation. It may -depend on generic source and structured-completion contracts and may consume -its scoped assets under `assets/generic/`, but it must not import a production -domain or encode D&D identity rules. - -The shared core owns: - -- validation and defensive copying of its inputs; -- candidate eligibility and stable input ordering; -- assignment of model-visible request-local candidate IDs; -- construction of bounded candidate and source-context materials; -- invocation of the configured structured LLM prompt; -- the private duplicate-group response schema; -- exact resolution of response IDs to the candidates visible in that request; -- rejection of malformed, unknown, repeated, ambiguous, or overlapping - proposals; -- canonical ordering of accepted groups and group members; -- construction of an immutable reconciliation plan; -- shared retry, fallback, cancellation, and invalid-output classification; -- generic consolidation and provenance mechanics that are demonstrably common - across typed consumers; and -- metadata and checkpoint fingerprints for shared prompts, schemas, and policy - versions. - -The core should expose stable behavior rather than a collection of unrelated -helpers. Its API should make the safe path direct: a caller supplies typed -candidates and a narrow domain policy, and receives either a deterministic -typed result or a validated reconciliation plan that can only be applied -through the typed policy boundary. - -### Typed Artifact-Family Adapter - -Each consuming artifact family owns a typed adapter or policy that supplies the -irreducibly domain-specific behavior: - -- projection of merged typed records into eligible reconciliation candidates; -- the semantic identity scope being reconciled; -- an optional domain semantic-policy prompt asset; -- canonical-field selection beyond choosing the supplied canonical member; -- domain field and evidence consolidation rules; -- durable ID derivation; -- domain warning scopes, reason codes, and messages; and -- domain postconditions and fallback behavior not covered by the shared core. - -The adapter must not parse untyped durable JSON or weaken the artifact codec's -exact Go type. The core must not use reflection to infer domain fields. Shared -typed consolidation support may use Go generics and a deliberately small policy -interface where that removes demonstrated duplication without hiding domain -rules. - -### Pipeline Boundary - -Semantic reconciliation remains an implementation of the normalize stage. It -does not add a pipeline stage, hide normalization inside extraction, or create -an arbitrary workflow edge. Existing typed normalizer registrations remain the -public configuration boundary. - -The initial implementation does not register one universal -`generic/deduplication` module key. A module key cannot safely accept arbitrary -artifact kinds under the current exact typed-registration contract. Domain -registrars instead instantiate the shared strategy for their own artifact -types. - -## Request-Local Candidate Protocol - -### Candidate Presentation - -After deterministic preprocessing and eligibility filtering, the core assigns -the candidates visible to one completion request contiguous integer -`candidate_id` values beginning with `1`. The mapping is owned by that request -and retains the original typed candidate and input position internally. - -Each model-facing candidate includes its `candidate_id`, contextual display -name or label, and the evidence and context needed for semantic judgment. In -the initial source-backed implementation, that context consists of validated -source-reference ranges and bounded, source-ordered transcript windows. -Application-owned durable entity IDs are never included. - -Candidate IDs: - -- are identifiers for prompt selections, not entity identities; -- have meaning only within one structured completion request; -- restart for each request or future batch; -- never enter a durable artifact or public schema; -- are not used to derive durable IDs; and -- must be resolved through the core's retained request-local mapping. - -Only model-visible eligible candidates receive IDs. Filtering must not produce -gaps that increase model burden or reveal unrelated internal ordering. - -### Proposal Response - -The private structured response contains an ordered `duplicate_groups` array. -Each group contains: - -- `candidate_ids`: at least two distinct supplied integer IDs; and -- `canonical_candidate_id`: one supplied ID that is also a member of that - group. - -The model does not return candidate names, source ranges, replacement records, -durable IDs, or synthesized canonical values. Choosing a canonical candidate -means selecting one supplied member; typed deterministic code constructs the -resulting record. - -The response schema should reject unknown fields and require every defined -field. Application validation remains authoritative and must additionally -reject out-of-range IDs, repeated members, a canonical ID outside its group, -and any candidate appearing in more than one group. Schema validation is not a -substitute for these semantic checks. - -### Deterministic Plan - -The core resolves accepted IDs to internal candidates, orders members by their -original deterministic positions, and orders groups by their earliest member. -Candidates omitted from the response remain distinct. Invalid groups are never -partially applied. - -The reconciliation plan retains enough internal provenance for typed -application, warning generation, debugging, and validation without exposing -request-local IDs as durable identity. - -## Prompt Policy - -### Mandatory Shared Protocol - -The core owns a shared prompt fragment that defines the response protocol and -non-negotiable safety behavior. It instructs the model to: - -- identify only well-supported groups that denote the same underlying entity; -- preserve candidates that are merely similar or uncertain; -- return only supplied candidate IDs; -- select one supplied group member as canonical; -- omit uncertain groups; and -- invent no candidates, evidence, attributes, identities, or replacements. - -This protocol fragment and the private schema are not replaceable by a domain -adapter. Keeping them shared ensures identical mechanics across consumers and -provides one prompt prefix for review, testing, and provider caching. - -### Generic Semantic Default - -The core provides a conservative generic semantic-policy fragment suitable for -an artifact family whose notion of entity identity is adequately conveyed by -its candidate labels and evidence. It asks whether candidates refer to the same -underlying entity and treats uncertainty as a reason not to collapse them. - -Use of the generic default is explicit in the typed adapter. It is not an -implicit fallback for an adapter that failed to declare its policy. - -### Domain Semantic Policy - -An artifact family may select a domain-owned semantic-policy fragment in place -of the generic semantic fragment. The replacement defines only domain judgment -and canonical-member preferences; it does not replace the shared protocol, -response schema, or deterministic safety rules. - -The existing NPC, item, and location policies remain domain-owned because they -encode meaningful distinctions among people, item types or unique -designations, currency denominations, parent and child places, and same-name -physical locations. Their prompt manifests should select the shared protocol -and their local semantic fragment while using the same generic response -schema. - -Prompt selection is initially an implementation-time artifact-family choice. -This work does not add arbitrary operator-supplied prompt paths or configuration -that can replace reconciliation safety policy. - -## Context, Bounds, And Model Invocation - -The first shared core is intentionally scoped to candidates grounded in a -Notarius source document. It validates candidate source references, creates -bounded windows around their evidence, preserves source order, and supplies -the model with the relationship between each candidate ID and its evidence. -Source ranges are model input but are never model output. - -The core skips the LLM call when fewer than two candidates remain eligible. It -must also impose explicit candidate-count and rendered-context bounds. When an -input cannot be reconciled safely within those bounds, normalization preserves -the deterministic preprocessed result and produces bounded diagnostics under -the consuming module's established fallback policy. It must not silently -process arbitrary fixed-size slices that could separate duplicates. - -Every completion uses the injected scheduled structured-LLM client, propagated -profile and session ID, cancellation, and the normalizer retry contract. -Invalid structured output or an unusable proposal follows the existing -retry-then-deterministic-fallback model. Transport and provider failures remain -execution errors rather than being silently converted into semantic absence. - -## Deterministic Typed Application - -The model never mutates the artifact. The shared core and typed adapter apply -only fully validated groups. - -Application must preserve these invariants: - -- no ungrouped candidate is inserted, removed, or changed by semantic - reconciliation; -- every accepted group produces exactly one typed output record; -- canonical display fields come from a supplied group member unless a domain - policy explicitly performs a deterministic transformation; -- all required provenance from group members is retained and canonicalized; -- output order follows the earliest contributing deterministic input position; -- durable IDs are recomputed by the domain identity policy after - consolidation; -- caller-owned input and request material are never mutated; -- warnings identify every collapsed group using domain-owned scopes and reason - codes; and -- warning volume remains bounded. - -Any shared typed application helper must make these invariants structural while -leaving domain field merging and ID derivation explicit. - -## Target D&D Consumers - -### NPC Registry - -NPC normalization uses the shared core while preserving its individual-person -identity semantics, proper-name canonicalization policy, evidence union, -deterministic NPC ID derivation, warning behavior, retry, and fallback result. -Its model response uses integer candidate selection rather than copied -contextual selectors. - -### Item Registry - -Item normalization uses the shared core while preserving the distinction -between item types and unique designations, distinct currency denominations, -non-inference of item properties or uniqueness, item identity derivation, -warnings, and fallback behavior. - -### Location Registry - -Location normalization uses the shared core while preserving its -physical-place identity semantics, treatment of parent and child places and -same-name places, source-reference-dependent durable IDs, warnings, and -fallback behavior. - -The migrations must retain the existing durable artifact schemas and public -module keys. Prompt, schema, policy, and implementation fingerprint changes -must invalidate only the affected normalization checkpoints through the normal -checkpoint identity mechanism. - -The target tree contains no superseded D&D-specific selector-copy response -schema, shared reconciliation implementation, or obsolete prompt assets. It -does not retain compatibility shims for the private pre-release LLM response -contract. - -## Verification Strategy - -Tests should protect the reconciliation contract and realistic failure modes, -not private helper structure or exact prompt length. - -The shared core warrants focused behavioral coverage for: - -- contiguous ID assignment after eligibility filtering; -- absence of durable IDs from model inputs and absence of evidence ranges from - model outputs; -- exact ID resolution and rejection of zero, negative, unknown, repeated, and - overlapping IDs; -- canonical membership and minimum group size; -- deterministic group and output ordering regardless of response order; -- preservation of ungrouped candidates and complete provenance; -- ownership and defensive-copy guarantees; -- cancellation, invalid structured output, retry, and fallback behavior; -- candidate and context bounds; and -- offline prompt/schema registration and structured response decoding. - -Use table-driven or property-oriented tests where they efficiently protect -group-partition and preservation invariants. Fuzzing is appropriate for the -pure proposal assessor if it remains fast and deterministic. Do not add tests -that merely freeze prompt text, message counts, asset hashes, private constant -values, or implementation call choreography. - -Each migrated D&D normalizer retains focused tests for its domain identity, -consolidation, warnings, durable IDs, and fallback behavior. A small number of -integration tests should prove that production registration supplies the -shared schema and selected prompt policy. Live-model evaluation remains a -human review tool and is not part of the default offline test suite. - -## Architectural Decisions And Documentation - -The target documentation set includes an accepted ADR applying -[ADR-0012](../adr/0012-resolve-opaque-entity-identifiers-deterministically.md) -to semantic reconciliation. That decision records: - -- request-local ordinal candidate handles as the standard reconciliation - selection mechanism; -- why requiring the model to reproduce contextual selectors is unnecessary - and error-prone; -- the mandatory shared protocol plus generic or domain semantic-policy - composition; -- LLM proposal versus deterministic application ownership; -- the typed adapter boundary; and -- rejected alternatives, including durable IDs, name-only selection, - model-synthesized replacement records, and arbitrary untyped normalization. - -ADR-0012's accepted decision text remains unchanged. The new ADR cites it and -provides the concrete justification it requires for request-local labels. - -In the target state, architecture and internal documentation: - -- distinguish artifact families from configured stage modules; -- identify the shared reconciliation core and its dependency direction; -- document the generic prompt protocol and domain semantic-policy ownership; -- document the integer candidate protocol and deterministic safety boundary; -- describe the three migrated registry normalizers accurately; and -- remove descriptions of contextual-selector response copying. - -Until then, this roadmap remains the canonical description of the proposed -behavior; current-behavior documents must not describe it as implemented. - -## Non-Goals - -This work does not include: - -- a new pipeline stage or a compound module that combines extraction and - normalization; -- a universal configured normalizer for arbitrary artifact kinds or untyped - JSON; -- a physical reorganization of every D&D package around artifact-family - directories; -- operator-configurable arbitrary prompt assets or replacement of core safety - instructions; -- naive batching, cross-batch clustering, or unbounded reconciliation inputs; -- semantic deduplication of event artifacts whose identity dimensions are - already handled deterministically; -- alternate non-source context providers; -- changes to durable D&D artifact schemas or public module keys; or -- live provider calls in the default test suite. - -## Completion Criteria - -The feature is complete when: - -- one domain-neutral core owns candidate IDs, source context, prompt execution, - response assessment, retry classification, and common safe application - mechanics; -- the private response schema uses only integer candidate handles for member - and canonical selection; -- the generic safety protocol and conservative semantic default exist as - shared assets; -- domain adapters can explicitly select the generic semantic default or a - domain-owned semantic fragment without replacing core safety behavior; -- NPC, item, and location registry normalizers use the shared core and retain - their typed domain behavior and durable contracts; -- the model is not required to reproduce names, evidence ranges, durable IDs, - or replacement records in reconciliation output; -- invalid proposals cannot partially mutate or partially collapse an artifact; -- oversized inputs preserve deterministic results rather than being naively - divided; -- the superseded D&D-specific response and reconciliation path is removed; -- the new ADR and current-behavior documentation accurately reflect the - implemented boundary; and -- focused package tests and the repository-wide Go test and vet suites pass. diff --git a/internal/framework/semanticreconcile/preparation.go b/internal/framework/semanticreconcile/preparation.go index b1c7b59..6036566 100644 --- a/internal/framework/semanticreconcile/preparation.go +++ b/internal/framework/semanticreconcile/preparation.go @@ -189,37 +189,36 @@ func Prepare(document *source.SourceDocument, candidates []Candidate, limits Lim return result, nil } - candidateContent, err := json.Marshal(candidateInput{Candidates: views}) + candidateContent, withinLimit, err := marshalCandidateInput(views, limits.MaximumMaterialBytes) if err != nil { return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode candidate material: %w", err) } - if len(candidateContent) > limits.MaximumMaterialBytes { + if !withinLimit { result.disposition = LimitExceeded return result, nil } - intervals := make([]sourceInterval, 0) - cited := make([]bool, len(document.Units)) + contextIntervals := make([]sourceInterval, 0) + citedIntervals := make([]sourceInterval, 0) for _, candidate := range prepared { for _, interval := range candidate.intervals { - for position := interval.start; position <= interval.end; position++ { - cited[position] = true - } - intervals = append(intervals, sourceInterval{ + citedIntervals = append(citedIntervals, interval) + contextIntervals = append(contextIntervals, sourceInterval{ start: max(0, interval.start-limits.ContextRadius), end: min(len(document.Units)-1, interval.end+limits.ContextRadius), }) } } - windows, err := buildContextWindows(document.Units, coalesceIntervals(intervals), cited) + transcriptContent, withinLimit, err := marshalTranscriptInput( + document.Units, + coalesceIntervals(contextIntervals), + coalesceIntervals(citedIntervals), + limits.MaximumMaterialBytes-len(candidateContent), + ) if err != nil { return Preparation{}, fmt.Errorf("prepare semantic reconciliation: build transcript material: invalid source metadata") } - transcriptContent, err := json.Marshal(transcriptInput{Windows: windows}) - if err != nil { - return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode transcript material: %w", err) - } - if len(transcriptContent) > limits.MaximumMaterialBytes-len(candidateContent) { + if !withinLimit { result.disposition = LimitExceeded return result, nil } @@ -303,27 +302,93 @@ func coalesceIntervals(intervals []sourceInterval) []sourceInterval { return coalesced } -func buildContextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]transcriptWindow, error) { - windows := make([]transcriptWindow, 0, len(intervals)) - for _, interval := range intervals { - window := transcriptWindow{Units: make([]transcriptUnit, 0, interval.end-interval.start+1)} +func marshalCandidateInput(candidates []visibleCandidate, maximumBytes int) ([]byte, bool, error) { + content := make([]byte, 0, min(maximumBytes, 4096)) + var withinLimit bool + content, withinLimit = appendWithinLimit(content, maximumBytes, []byte(`{"candidates":[`)) + if !withinLimit { + return nil, false, nil + } + for index, candidate := range candidates { + encoded, err := json.Marshal(candidate) + if err != nil { + return nil, false, err + } + separator := []byte(nil) + if index > 0 { + separator = []byte(",") + } + content, withinLimit = appendWithinLimit(content, maximumBytes, separator, encoded) + if !withinLimit { + return nil, false, nil + } + } + content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}")) + return content, withinLimit, nil +} + +// marshalTranscriptInput retains at most maximumBytes while visiting source +// units in order. It deliberately serializes one unit at a time so an oversized +// request does not require a document-sized transcript copy before rejection. +func marshalTranscriptInput(units []source.SourceUnit, contextIntervals, citedIntervals []sourceInterval, maximumBytes int) ([]byte, bool, error) { + content := make([]byte, 0, min(maximumBytes, 4096)) + content, withinLimit := appendWithinLimit(content, maximumBytes, []byte(`{"windows":[`)) + if !withinLimit { + return nil, false, nil + } + + citedIndex := 0 + for windowIndex, interval := range contextIntervals { + separator := []byte(nil) + if windowIndex > 0 { + separator = []byte(",") + } + content, withinLimit = appendWithinLimit(content, maximumBytes, separator, []byte(`{"units":[`)) + if !withinLimit { + return nil, false, nil + } for position := interval.start; position <= interval.end; position++ { + for citedIndex < len(citedIntervals) && citedIntervals[citedIndex].end < position { + citedIndex++ + } + cited := citedIndex < len(citedIntervals) && citedIntervals[citedIndex].start <= position unit := units[position] metadata, err := source.CloneMetadata(unit.Metadata) if err != nil { - return nil, err + return nil, false, err } - window.Units = append(window.Units, transcriptUnit{ - ID: unit.ID, - Kind: unit.Kind, - Text: unit.Text, - Metadata: metadata, - Cited: cited[position], + encoded, err := json.Marshal(transcriptUnit{ + ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: metadata, Cited: cited, }) + if err != nil { + return nil, false, err + } + separator = nil + if position > interval.start { + separator = []byte(",") + } + content, withinLimit = appendWithinLimit(content, maximumBytes, separator, encoded) + if !withinLimit { + return nil, false, nil + } + } + content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}")) + if !withinLimit { + return nil, false, nil } - windows = append(windows, window) } - return windows, nil + content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}")) + return content, withinLimit, nil +} + +func appendWithinLimit(content []byte, maximumBytes int, parts ...[]byte) ([]byte, bool) { + for _, part := range parts { + if len(content) > maximumBytes || len(part) > maximumBytes-len(content) { + return content, false + } + content = append(content, part...) + } + return content, true } func newInputMaterial(name string, content []byte) contracts.LLMInputMaterial { diff --git a/internal/framework/semanticreconcile/preparation_test.go b/internal/framework/semanticreconcile/preparation_test.go index 5cc00c5..be2934c 100644 --- a/internal/framework/semanticreconcile/preparation_test.go +++ b/internal/framework/semanticreconcile/preparation_test.go @@ -277,6 +277,38 @@ func TestPrepareEnforcesCandidateLimitBeforeRenderingContext(t *testing.T) { } } +func TestPrepareStopsRenderingContextWhenMaterialLimitIsExceeded(t *testing.T) { + cycle := map[string]any{} + cycle["self"] = cycle + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 1, Text: strings.Repeat("oversized", 100)}, + {ID: 2, Text: "must not be inspected"}, + }} + candidates := candidatesForEveryUnit(document) + base, err := Prepare(document, candidates, Limits{ + ContextRadius: 0, + MaximumCandidates: len(candidates), + MaximumMaterialBytes: 10000, + }) + if err != nil || base.Disposition() != Ready { + t.Fatalf("Prepare(base) = disposition %v, error %v", base.Disposition(), err) + } + candidateBytes := len(base.Materials()[candidateInputName].Content) + document.Units[1].Metadata = cycle + + limited, err := Prepare(document, candidates, Limits{ + ContextRadius: 0, + MaximumCandidates: len(candidates), + MaximumMaterialBytes: candidateBytes + 64, + }) + if err != nil { + t.Fatalf("Prepare(limited) error = %v; rendering should stop at the material bound", err) + } + if limited.Disposition() != LimitExceeded || len(limited.Materials()) != 0 { + t.Fatalf("Prepare(limited) = disposition %v, materials %#v, want bounded skip", limited.Disposition(), limited.Materials()) + } +} + func TestPrepareAcceptsExactCombinedByteLimitAndSkipsOneOver(t *testing.T) { document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ {ID: 1, Text: "one"}, diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer.go b/internal/modules/dnd/normalize/itemregistry/normalizer.go index 57f2fc1..9b236f5 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer.go @@ -102,11 +102,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context error before normalize: %w", err) } + if req.Source == nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil") + } order := shared.NewSourceRefOrder(req.Source) records, warnings := preprocessRecords(req.MergeOutput.Value, order) deterministic := recordList(records) - if len(records) < 2 || req.Source == nil { + if len(records) < 2 { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil } diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer_test.go b/internal/modules/dnd/normalize/itemregistry/normalizer_test.go index f6bf18f..a504e3c 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer_test.go @@ -47,6 +47,16 @@ func TestModuleContractAndMetadata(t *testing.T) { } } +func TestNormalizeRejectsNilSourceDocument(t *testing.T) { + _, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize( + context.Background(), + contracts.TypedNormalizeRequest[dnd.ItemRegistry]{}, + ) + if err == nil || !strings.Contains(err.Error(), "source document must not be nil") { + t.Fatalf("Normalize() error = %v, want nil source rejection", err) + } +} + func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) { doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ {ID: 1, Text: "The rope is secured."}, @@ -380,7 +390,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz return normalizer } func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { - return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}} + return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{ + Source: &source.SourceDocument{}, + MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}, + } } func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { request := normalizeRequest(value) diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer.go b/internal/modules/dnd/normalize/locationregistry/normalizer.go index b75a13c..0324bb0 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer.go @@ -103,11 +103,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context error before normalize: %w", err) } + if req.Source == nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("source document must not be nil") + } order := shared.NewSourceRefOrder(req.Source) records, warnings := preprocessRecords(req.MergeOutput.Value, order) deterministic := recordList(records) - if len(records) < 2 || req.Source == nil { + if len(records) < 2 { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil } diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer_test.go b/internal/modules/dnd/normalize/locationregistry/normalizer_test.go index 59db479..4b66a17 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer_test.go @@ -44,6 +44,16 @@ func TestModuleContractAndMetadata(t *testing.T) { } } +func TestNormalizeRejectsNilSourceDocument(t *testing.T) { + _, err := newNormalizer(t, &recordingLocationNormalizerClient{}).Normalize( + context.Background(), + contracts.TypedNormalizeRequest[dnd.LocationRegistry]{}, + ) + if err == nil || !strings.Contains(err.Error(), "source document must not be nil") { + t.Fatalf("Normalize() error = %v, want nil source rejection", err) + } +} + func TestNormalizePreparesOnlyExactDuplicatesAndRetainsSameNameAndNestedPlaces(t *testing.T) { input := dnd.LocationRegistry{Locations: []dnd.Location{ {Name: " The Tavern ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}, diff --git a/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go b/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go index 0d4a54b..39fcdf5 100644 --- a/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go +++ b/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go @@ -41,7 +41,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz return normalizer } func normalizeRequest(value dnd.LocationRegistry) contracts.TypedNormalizeRequest[dnd.LocationRegistry] { - return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value}} + return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{ + Source: &source.SourceDocument{}, + MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value}, + } } func normalizeRequestWithSource(value dnd.LocationRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.LocationRegistry] { request := normalizeRequest(value) diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer.go b/internal/modules/dnd/normalize/npcregistry/normalizer.go index a4ce414..d2c5591 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer.go @@ -102,11 +102,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("context error before normalize: %w", err) } + if req.Source == nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("source document must not be nil") + } order := shared.NewSourceRefOrder(req.Source) records, warnings := preprocessRecords(req.MergeOutput.Value, order) deterministic := recordList(records) - if len(records) < 2 || req.Source == nil { + if len(records) < 2 { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil } diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer_test.go b/internal/modules/dnd/normalize/npcregistry/normalizer_test.go index f079d19..672b524 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer_test.go @@ -126,6 +126,9 @@ func TestNormalizePreservesInvalidCandidatesAndDoesNotAliasInput(t *testing.T) { func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) { normalizer := newNormalizer(t, &recordingNPCNormalizerClient{}) + if _, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[dnd.NPCRegistry]{}); err == nil || !strings.Contains(err.Error(), "source document must not be nil") { + t.Fatalf("nil source Normalize() error = %v", err) + } result, err := normalizer.Normalize(context.Background(), normalizeRequest(dnd.NPCRegistry{NPCs: nil})) if err != nil || result.Value.NPCs != nil { t.Fatalf("nil list result = %#v, error = %v", result.Value, err) @@ -177,7 +180,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz } func normalizeRequest(value dnd.NPCRegistry) contracts.TypedNormalizeRequest[dnd.NPCRegistry] { - return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value}} + return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{ + Source: &source.SourceDocument{}, + MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value}, + } } func normalizeRequestWithSource(value dnd.NPCRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {