Compare commits
5 Commits
2c98763b9b
...
3eb68baca6
| Author | SHA1 | Date | |
|---|---|---|---|
| 3eb68baca6 | |||
| ae97adb8b0 | |||
| 79b9fffcaf | |||
| be22852daa | |||
| f5107045c3 |
@@ -285,6 +285,7 @@ production validators do not call the LLM and must not set `llm_profile`.
|
|||||||
| extract | `dnd/spells` | Extracts typed D&D spell-list artifacts. |
|
| extract | `dnd/spells` | Extracts typed D&D spell-list artifacts. |
|
||||||
| merge | `appendorder` | Combines typed artifacts in chunk order. |
|
| merge | `appendorder` | Combines typed artifacts in chunk order. |
|
||||||
| normalize | `noop` | Passes merged typed artifacts through unchanged. |
|
| normalize | `noop` | Passes merged typed artifacts through unchanged. |
|
||||||
|
| normalize | `dnd/spells` | Deterministically canonicalizes and de-duplicates typed D&D spell-list artifacts. |
|
||||||
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
||||||
|
|
||||||
## Implemented Production Validators
|
## Implemented Production Validators
|
||||||
@@ -300,7 +301,8 @@ production validators do not call the LLM and must not set `llm_profile`.
|
|||||||
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
|
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
|
||||||
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
|
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
|
||||||
|
|
||||||
The production default chain for the `dnd/spells` extractor is:
|
The production default chain for `dnd/spells` is used for both its extract and
|
||||||
|
normalize stages:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
validators:
|
validators:
|
||||||
@@ -342,6 +344,13 @@ not allow multiple files. Its format is defined in the
|
|||||||
The extractor uses campaign references only as supporting disambiguation
|
The extractor uses campaign references only as supporting disambiguation
|
||||||
material; spell casts still must be present in the source transcript.
|
material; spell casts still must be present in the source transcript.
|
||||||
|
|
||||||
|
The `dnd/spells` normalizer declares the same optional `spell_catalog` slot.
|
||||||
|
When an overlay is used, bind it independently under
|
||||||
|
`artifacts.<lane>.normalize.references.spell_catalog`; normalize-stage
|
||||||
|
references are local to that stage and are not inherited from extraction. The
|
||||||
|
normalizer uses the embedded SRD catalog when no normalize-stage overlay is
|
||||||
|
bound.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
The `output`, `cache`, and `debug` top-level fields select independent physical
|
The `output`, `cache`, and `debug` top-level fields select independent physical
|
||||||
|
|||||||
@@ -68,6 +68,39 @@ Reference slot keys and accepted file types are defined in
|
|||||||
supporting disambiguation material, not source evidence, and are not
|
supporting disambiguation material, not source evidence, and are not
|
||||||
addressable through `source_refs`.
|
addressable through `source_refs`.
|
||||||
|
|
||||||
|
## Normalization Behavior
|
||||||
|
|
||||||
|
When the `dnd/spells` normalizer is selected, each recognized spell name is
|
||||||
|
rewritten to the effective catalog's canonical display name. Lookup uses the
|
||||||
|
catalog's case-insensitive, whitespace-normalizing, apostrophe-normalizing, and
|
||||||
|
alias rules. Unknown names are preserved exactly for the normalize validators;
|
||||||
|
the normalizer does not guess or apply fuzzy matching.
|
||||||
|
|
||||||
|
Each cast's `source_refs` is copied, sorted by exact `source_id`,
|
||||||
|
`start_unit_id`, and `end_unit_id`, and stripped of exact structural
|
||||||
|
duplicates. Adjacent or overlapping ranges are not merged, and the normalizer
|
||||||
|
does not synthesize references or change their boundaries.
|
||||||
|
|
||||||
|
After those per-cast changes, duplicate identity requires the same canonical
|
||||||
|
spell name, the same caster after case folding and whitespace normalization,
|
||||||
|
and the same complete, non-empty set of source references valid for the source
|
||||||
|
document. Only the first occurrence is retained, in stable order. Its caster,
|
||||||
|
effect, narrative description, and canonical references are preserved without
|
||||||
|
prose merging or source union. Unknown names, empty or invalid evidence, and
|
||||||
|
casts with different evidence remain separate.
|
||||||
|
|
||||||
|
Mutation and duplicate decisions are returned through the normal warnings
|
||||||
|
surface. Warning scopes use the merged input index, such as `spell_casts[0]`,
|
||||||
|
so they remain meaningful even when a later duplicate is removed. The
|
||||||
|
normalizer uses these reason codes:
|
||||||
|
|
||||||
|
| Reason code | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `spell_name_canonicalized` | A catalog lookup replaced an input name with its canonical display name. |
|
||||||
|
| `spell_name_unresolved` | A name was not found in the effective catalog and was retained unchanged. |
|
||||||
|
| `source_references_normalized` | Reference order changed or exact duplicate references were removed. |
|
||||||
|
| `duplicate_spell_cast_collapsed` | A later cast matched the retained cast's complete duplicate identity. |
|
||||||
|
|
||||||
## Manifest Metadata
|
## Manifest Metadata
|
||||||
|
|
||||||
The extractor adds prompt and response-schema provenance under the artifact lane
|
The extractor adds prompt and response-schema provenance under the artifact lane
|
||||||
@@ -88,6 +121,11 @@ manifest metadata:
|
|||||||
"catalog_base_id": "dnd-5e-2014-srd-spells",
|
"catalog_base_id": "dnd-5e-2014-srd-spells",
|
||||||
"catalog_digest": "sha256:...",
|
"catalog_digest": "sha256:...",
|
||||||
"catalog_overlay_ids": ["campaign.example"]
|
"catalog_overlay_ids": ["campaign.example"]
|
||||||
|
},
|
||||||
|
"normalizer": {
|
||||||
|
"catalog_base_id": "dnd-5e-2014-srd-spells",
|
||||||
|
"catalog_digest": "sha256:...",
|
||||||
|
"catalog_overlay_ids": ["campaign.example"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -96,6 +134,7 @@ manifest metadata:
|
|||||||
`catalog_digest` identifies the effective semantic catalog, while
|
`catalog_digest` identifies the effective semantic catalog, while
|
||||||
`catalog_overlay_ids` is sorted and empty for a base-only configuration. Raw
|
`catalog_overlay_ids` is sorted and empty for a base-only configuration. Raw
|
||||||
prompt, schema, catalog, alias, and local overlay-file content are not
|
prompt, schema, catalog, alias, and local overlay-file content are not
|
||||||
included in manifest metadata. Overlay origin, media type, byte size, and raw
|
included in manifest metadata. The `normalizer` metadata uses the same catalog
|
||||||
digest are recorded separately in the manifest's reference provenance; see
|
identity fields when that module is selected. Overlay origin, media type, byte
|
||||||
the [JSON output contract](json-output.md#manifestjson).
|
size, and raw digest are recorded separately in the manifest's reference
|
||||||
|
provenance; see the [JSON output contract](json-output.md#manifestjson).
|
||||||
|
|||||||
@@ -186,6 +186,30 @@ function.
|
|||||||
The normalizer returns the merged domain value unchanged and is reusable for
|
The normalizer returns the merged domain value unchanged and is reusable for
|
||||||
any registered artifact type.
|
any registered artifact type.
|
||||||
|
|
||||||
|
### `internal/modules/dnd/normalize/spells`
|
||||||
|
|
||||||
|
The typed spell normalizer resolves the optional `spell_catalog` reference into
|
||||||
|
the same immutable SRD-plus-overlay effective catalog used by spell extraction
|
||||||
|
and catalog validation. It performs no LLM calls. For each spell cast it
|
||||||
|
canonicalizes recognized names using the catalog's case, whitespace,
|
||||||
|
apostrophe, and alias rules; sorts source references by source identity and
|
||||||
|
unit boundaries; removes only exact reference duplicates; and emits bounded,
|
||||||
|
scoped warnings for each mutation or unresolved name.
|
||||||
|
|
||||||
|
After those per-cast changes, it collapses only casts with the same canonical
|
||||||
|
spell, case-folded and whitespace-normalized caster, and complete non-empty
|
||||||
|
valid source-reference set. It retains the first occurrence and its caster,
|
||||||
|
effect, narrative description, and stable order. Unknown names, empty or
|
||||||
|
invalid evidence, and adjacent or overlapping but different ranges remain
|
||||||
|
unchanged for validation.
|
||||||
|
|
||||||
|
The normalizer exposes the effective catalog digest as its independently scoped
|
||||||
|
`effective_catalog` checkpoint fingerprint and reports catalog base ID, digest,
|
||||||
|
and overlay IDs as manifest metadata. Catalog contents, reference paths, and
|
||||||
|
raw overlay bytes are not included in either surface. The normalize-stage
|
||||||
|
reference is stage-local, so an overlay-capable pipeline binds the catalog
|
||||||
|
independently for extraction and normalization.
|
||||||
|
|
||||||
## Output Encoder
|
## Output Encoder
|
||||||
|
|
||||||
### `internal/modules/generic/output/json`
|
### `internal/modules/generic/output/json`
|
||||||
@@ -240,7 +264,7 @@ complete framework registry set and one LLM asset registry. It invokes
|
|||||||
`internal/modules/seriatim/register`, and `internal/modules/dnd/register` in
|
`internal/modules/seriatim/register`, and `internal/modules/dnd/register` in
|
||||||
that order, then exposes the matching catalog for resolution. The generic and
|
that order, then exposes the matching catalog for resolution. The generic and
|
||||||
Seriatim registrars own their production leaf registrations. The D&D registrar
|
Seriatim registrars own their production leaf registrations. The D&D registrar
|
||||||
owns D&D leaf registrations, the spell default-validator chain, and D&D
|
owns D&D leaf registrations, the typed spell default-validator chains, and D&D
|
||||||
prompt/schema asset collection.
|
prompt/schema asset collection.
|
||||||
|
|
||||||
Concrete implementation packages do not import generic implementation
|
Concrete implementation packages do not import generic implementation
|
||||||
|
|||||||
@@ -65,8 +65,8 @@ run-local construction closures. Preparation injects shared dependencies and
|
|||||||
constructs input, chunk, validators, ordered lanes, and output before source
|
constructs input, chunk, validators, ordered lanes, and output before source
|
||||||
parsing. Production modules use strict construction-time option decoding, and
|
parsing. Production modules use strict construction-time option decoding, and
|
||||||
LLM-backed modules retain the injected shared client. The D&D family registers
|
LLM-backed modules retain the injected shared client. The D&D family registers
|
||||||
the canonical `dnd/spell-list` codec, typed spell extractor and validators, and
|
the canonical `dnd/spell-list` codec, typed spell extractor, normalizer, and
|
||||||
kind-specific generic merge and normalize strategies; generic JSON validators
|
validators, plus kind-specific generic merge strategies; generic JSON validators
|
||||||
use the serialized-validation contract. The runner executes lanes through
|
use the serialized-validation contract. The runner executes lanes through
|
||||||
private exact-type-checked closures, coordinates extract results independently
|
private exact-type-checked closures, coordinates extract results independently
|
||||||
of completion timing, and serializes artifacts only through their codec at
|
of completion timing, and serializes artifacts only through their codec at
|
||||||
@@ -90,6 +90,7 @@ Configuration. The implemented module packages are:
|
|||||||
| `internal/modules/dnd/spells/catalog` | Embeds and validates the versioned D&D 5e 2014 SRD catalog, composes optional overlays, and provides immutable effective lookup. |
|
| `internal/modules/dnd/spells/catalog` | Embeds and validates the versioned D&D 5e 2014 SRD catalog, composes optional overlays, and provides immutable effective lookup. |
|
||||||
| `internal/modules/generic/merge/appendorder` | Combines accepted extraction results in chunk order. |
|
| `internal/modules/generic/merge/appendorder` | Combines accepted extraction results in chunk order. |
|
||||||
| `internal/modules/generic/normalize/noop` | Preserves accepted merged output. |
|
| `internal/modules/generic/normalize/noop` | Preserves accepted merged output. |
|
||||||
|
| `internal/modules/dnd/normalize/spells` | Canonicalizes catalog-backed spell names and exact source references, conservatively collapses duplicate casts, and reports deterministic warnings and independently scoped catalog checkpoint identity. |
|
||||||
| `internal/modules/generic/output/json` | Encodes manifests, lane payloads, warnings, and rejections as logical JSON files. |
|
| `internal/modules/generic/output/json` | Encodes manifests, lane payloads, warnings, and rejections as logical JSON files. |
|
||||||
|
|
||||||
`internal/modules/dnd/shared` owns reusable D&D prompt fragments,
|
`internal/modules/dnd/shared` owns reusable D&D prompt fragments,
|
||||||
|
|||||||
@@ -2,14 +2,8 @@
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
<<<<<<< HEAD
|
Implemented deterministic feature. Implementation details are maintained
|
||||||
Proposed as the next D&D pipeline milestone. This feature completes the first
|
separately in `docs/roadmap/implementation.md`.
|
||||||
domain-specific normalize stage for the spell pipeline before work begins on
|
|
||||||
NPC and combat-turn artifacts.
|
|
||||||
=======
|
|
||||||
Accepted target state. Implementation details are maintained separately in
|
|
||||||
`docs/roadmap/implementation.md`.
|
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
## Objective
|
## Objective
|
||||||
|
|
||||||
@@ -18,13 +12,8 @@ that emits canonical catalog names, removes only clearly identical duplicate
|
|||||||
casts, preserves source provenance, and makes every mutation visible through
|
casts, preserves source provenance, and makes every mutation visible through
|
||||||
scoped warnings.
|
scoped warnings.
|
||||||
|
|
||||||
<<<<<<< HEAD
|
|
||||||
The milestone should improve the consistency of durable spell output without
|
|
||||||
adding another LLM call or introducing fuzzy repair policy.
|
|
||||||
=======
|
|
||||||
The normalizer improves the consistency of durable spell output without adding
|
The normalizer improves the consistency of durable spell output without adding
|
||||||
another LLM call or introducing fuzzy repair policy.
|
another LLM call or introducing fuzzy repair policy.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
## Target Behavior
|
## Target Behavior
|
||||||
|
|
||||||
@@ -43,24 +32,16 @@ depend on extractor-private state.
|
|||||||
The normalizer contributes its effective catalog digest through
|
The normalizer contributes its effective catalog digest through
|
||||||
`pipeline.CheckpointFingerprintProvider`. Changing the embedded catalog,
|
`pipeline.CheckpointFingerprintProvider`. Changing the embedded catalog,
|
||||||
composition policy, or normalize-stage overlay therefore invalidates reusable
|
composition policy, or normalize-stage overlay therefore invalidates reusable
|
||||||
<<<<<<< HEAD
|
|
||||||
normalize checkpoints.
|
|
||||||
=======
|
|
||||||
normalize checkpoints. It also records catalog base ID, digest, and overlay IDs
|
normalize checkpoints. It also records catalog base ID, digest, and overlay IDs
|
||||||
as manifest metadata. Neither identity surface includes catalog contents or
|
as manifest metadata. Neither identity surface includes catalog contents or
|
||||||
reference paths.
|
reference paths.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
### Canonical spell names
|
### Canonical spell names
|
||||||
|
|
||||||
For every spell cast, look up the extracted name using the effective catalog's
|
For every spell cast, look up the extracted name using the effective catalog's
|
||||||
existing case, whitespace, apostrophe, and alias rules. Replace a recognized
|
existing case, whitespace, apostrophe, and alias rules. Replace a recognized
|
||||||
value with its canonical display name. This is the only spell-name repair in
|
value with its canonical display name. This is the only spell-name repair in
|
||||||
<<<<<<< HEAD
|
|
||||||
the initial feature.
|
|
||||||
=======
|
|
||||||
this feature.
|
this feature.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
Do not use edit distance, phonetic matching, model judgment, or another fuzzy
|
Do not use edit distance, phonetic matching, model judgment, or another fuzzy
|
||||||
heuristic. If a value does not resolve, retain it unchanged and emit a scoped
|
heuristic. If a value does not resolve, retain it unchanged and emit a scoped
|
||||||
@@ -69,12 +50,8 @@ acceptance or rejection.
|
|||||||
|
|
||||||
Emit a warning for each changed spell name. Diagnostics should identify the
|
Emit a warning for each changed spell name. Diagnostics should identify the
|
||||||
artifact index and the original and canonical values without modifying other
|
artifact index and the original and canonical values without modifying other
|
||||||
<<<<<<< HEAD
|
|
||||||
fields.
|
|
||||||
=======
|
|
||||||
fields. Diagnostics are deterministic and bounded: user-controlled spell names
|
fields. Diagnostics are deterministic and bounded: user-controlled spell names
|
||||||
are truncated to 128 Unicode code points before display.
|
are truncated to 128 Unicode code points before display.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
### Source-reference normalization
|
### Source-reference normalization
|
||||||
|
|
||||||
@@ -84,12 +61,8 @@ overlapping ranges, because doing so could broaden the evidence attributed to
|
|||||||
an event.
|
an event.
|
||||||
|
|
||||||
The normalizer must not synthesize source references, alter source-unit
|
The normalizer must not synthesize source references, alter source-unit
|
||||||
<<<<<<< HEAD
|
|
||||||
boundaries, or use auxiliary references as evidence.
|
|
||||||
=======
|
|
||||||
boundaries, or use auxiliary references as evidence. Reordering references or
|
boundaries, or use auxiliary references as evidence. Reordering references or
|
||||||
removing exact duplicates emits a scoped warning for the affected input cast.
|
removing exact duplicates emits a scoped warning for the affected input cast.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
### Conservative duplicate collapse
|
### Conservative duplicate collapse
|
||||||
|
|
||||||
@@ -98,12 +71,8 @@ event only when all of the following match:
|
|||||||
|
|
||||||
- canonical spell name;
|
- canonical spell name;
|
||||||
- caster after case folding and whitespace normalization; and
|
- caster after case folding and whitespace normalization; and
|
||||||
<<<<<<< HEAD
|
|
||||||
- the complete canonical source-reference set.
|
|
||||||
=======
|
|
||||||
- the complete canonical source-reference set, which must be non-empty and
|
- the complete canonical source-reference set, which must be non-empty and
|
||||||
valid against the source document.
|
valid against the source document.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
Collapse each such group into its first occurrence, preserving stable pipeline
|
Collapse each such group into its first occurrence, preserving stable pipeline
|
||||||
order. Retain the first cast's caster, effect, and narrative description. Do
|
order. Retain the first cast's caster, effect, and narrative description. Do
|
||||||
@@ -116,14 +85,11 @@ the removed indices. Casts with different evidence remain distinct even when
|
|||||||
their spell and caster match. In particular, adjacency at a chunk or scene
|
their spell and caster match. In particular, adjacency at a chunk or scene
|
||||||
boundary is not sufficient evidence of duplication.
|
boundary is not sufficient evidence of duplication.
|
||||||
|
|
||||||
<<<<<<< HEAD
|
|
||||||
=======
|
|
||||||
Unknown spell names, empty evidence sets, and invalid source references cannot
|
Unknown spell names, empty evidence sets, and invalid source references cannot
|
||||||
establish duplicate identity. They remain in the artifact for the configured
|
establish duplicate identity. They remain in the artifact for the configured
|
||||||
normalize validators to accept or reject. A duplicate warning displays at most
|
normalize validators to accept or reject. A duplicate warning displays at most
|
||||||
20 removed input indices and reports the exact omitted count.
|
20 removed input indices and reports the exact omitted count.
|
||||||
|
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
### Production composition and validation
|
### Production composition and validation
|
||||||
|
|
||||||
Register the typed spell normalizer in the D&D family and make it the default
|
Register the typed spell normalizer in the D&D family and make it the default
|
||||||
@@ -141,40 +107,34 @@ without a catalog reference.
|
|||||||
|
|
||||||
## Acceptance Criteria
|
## Acceptance Criteria
|
||||||
|
|
||||||
- Base and overlay aliases are written using canonical catalog display names.
|
- [x] Base and overlay aliases are written using canonical catalog display names.
|
||||||
- Unknown names are never guessed, deleted, or silently changed.
|
- [x] Unknown names are never guessed, deleted, or silently changed.
|
||||||
- Source references are deterministically sorted and exactly deduplicated.
|
- [x] Source references are deterministically sorted and exactly deduplicated.
|
||||||
- Only casts with matching canonical spell, normalized caster, and identical
|
- [x] Only casts with matching canonical spell, normalized caster, and identical
|
||||||
<<<<<<< HEAD
|
|
||||||
evidence sets collapse; near or adjacent events remain separate.
|
|
||||||
- Normalization preserves the first cast's non-name fields and stable order.
|
|
||||||
- Every name change and duplicate collapse emits a scoped warning.
|
|
||||||
=======
|
|
||||||
non-empty valid evidence sets collapse; near, invalid, unknown, or adjacent
|
non-empty valid evidence sets collapse; near, invalid, unknown, or adjacent
|
||||||
events remain separate.
|
events remain separate.
|
||||||
- Normalization preserves the first cast's non-name fields and stable order.
|
- [x] Normalization preserves the first cast's non-name fields and stable order.
|
||||||
- Every name change, unresolved name, source-reference mutation, and duplicate
|
- [x] Every name change, unresolved name, source-reference mutation, and duplicate
|
||||||
collapse emits a scoped warning.
|
collapse emits a scoped warning.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
- [x] Extract, normalize, and catalog-validation catalog identities agree for the
|
||||||
- Extract, normalize, and catalog-validation catalog identities agree for the
|
|
||||||
same bound references, and catalog changes invalidate normalize checkpoints.
|
same bound references, and catalog changes invalidate normalize checkpoints.
|
||||||
- The maintained production pipeline and current-behavior documentation use
|
- [x] The maintained production pipeline and current-behavior documentation use
|
||||||
the D&D normalizer without changing the durable artifact schema.
|
the D&D normalizer without changing the durable artifact schema.
|
||||||
|
|
||||||
## Evaluation
|
## Evaluation
|
||||||
|
|
||||||
Maintain a small human-reviewed set of representative spell outputs covering
|
Status: pending qualitative review. No approved human-reviewed transcript
|
||||||
canonical names, aliases, repeated casts, duplicate model output, and scene
|
corpus is available locally under repository policy, so no sensitive transcript
|
||||||
boundaries. Use it to review normalization behavior and warnings, not as a
|
content is committed and no claims about real transcript quality are made.
|
||||||
claim that LLM extraction is deterministically correct.
|
|
||||||
|
A compact deterministic fixture set covers canonical names, exact source
|
||||||
|
reference normalization, repeated casts, duplicate output, and distinct
|
||||||
|
evidence. It exercises the normalizer contract without requiring paid or
|
||||||
|
network LLM calls.
|
||||||
|
|
||||||
Record cases where likely duplicates remain because their evidence differs.
|
Record cases where likely duplicates remain because their evidence differs.
|
||||||
Those examples should inform later LLM-assisted deduplication work rather than
|
Those examples should inform later LLM-assisted deduplication work rather than
|
||||||
<<<<<<< HEAD
|
|
||||||
causing this deterministic milestone to adopt broader heuristics.
|
|
||||||
=======
|
|
||||||
causing this deterministic feature to adopt broader heuristics.
|
causing this deterministic feature to adopt broader heuristics.
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
|
|
||||||
## Deferred Work
|
## Deferred Work
|
||||||
|
|
||||||
@@ -186,10 +146,6 @@ causing this deterministic feature to adopt broader heuristics.
|
|||||||
- Repair-aware extraction retries or LLM-backed semantic validation.
|
- Repair-aware extraction retries or LLM-backed semantic validation.
|
||||||
- Changes to the spell artifact schema, including stable cast IDs.
|
- Changes to the spell artifact schema, including stable cast IDs.
|
||||||
|
|
||||||
<<<<<<< HEAD
|
|
||||||
An LLM-backed normalizer is explicitly outside this milestone. If later
|
|
||||||
=======
|
|
||||||
An LLM-backed normalizer is explicitly outside this feature. If later
|
An LLM-backed normalizer is explicitly outside this feature. If later
|
||||||
>>>>>>> 6fbdf6b (Add feature roadmap and implementation plan for D&D spell normalization module)
|
|
||||||
evaluation justifies one, shared spell-catalog prompt material should be
|
evaluation justifies one, shared spell-catalog prompt material should be
|
||||||
designed at that time around the actual normalization or repair request.
|
designed at that time around the actual normalization or repair request.
|
||||||
|
|||||||
@@ -7,11 +7,8 @@ not as committed release dates.
|
|||||||
|
|
||||||
## Near-Term D&D Pipeline
|
## Near-Term D&D Pipeline
|
||||||
|
|
||||||
### Solidify Spell Extraction
|
### Evaluate Spell Extraction And Normalization
|
||||||
|
|
||||||
- Implement the deterministic catalog-aware normalizer defined in
|
|
||||||
[D&D Spell Normalization](dnd-spell-normalization.md), including conservative
|
|
||||||
exact-evidence duplicate collapse.
|
|
||||||
- Evaluate ordinary extraction retries and the completed normalization path
|
- Evaluate ordinary extraction retries and the completed normalization path
|
||||||
against a human-reviewed transcript set before adding repair-aware retries or
|
against a human-reviewed transcript set before adding repair-aware retries or
|
||||||
an LLM-backed semantic validator.
|
an LLM-backed semantic validator.
|
||||||
|
|||||||
@@ -29,5 +29,11 @@ pipelines:
|
|||||||
extract:
|
extract:
|
||||||
module: dnd/spells
|
module: dnd/spells
|
||||||
retries: 2
|
retries: 2
|
||||||
|
# Overlay behavior binds the same catalog independently at each stage.
|
||||||
|
references:
|
||||||
|
spell_catalog: ./dnd-spells-catalog.json
|
||||||
|
normalize:
|
||||||
|
module: dnd/spells
|
||||||
|
# Normalize-stage references are local and must be bound explicitly.
|
||||||
references:
|
references:
|
||||||
spell_catalog: ./dnd-spells-catalog.json
|
spell_catalog: ./dnd-spells-catalog.json
|
||||||
|
|||||||
@@ -15,3 +15,5 @@ pipelines:
|
|||||||
artifacts:
|
artifacts:
|
||||||
spells:
|
spells:
|
||||||
extract: dnd/spells
|
extract: dnd/spells
|
||||||
|
# Base-only behavior: normalization uses the embedded SRD catalog.
|
||||||
|
normalize: dnd/spells
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -8,4 +8,4 @@ require (
|
|||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.40.0 // indirect
|
require golang.org/x/text v0.40.0
|
||||||
|
|||||||
231
internal/cli/assembled_spell_pipeline_contract_test.go
Normal file
231
internal/cli/assembled_spell_pipeline_contract_test.go
Normal file
@@ -0,0 +1,231 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||||
|
|
||||||
|
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||||
|
registries, resolved, extractor := assembledSpellPipeline(t, false)
|
||||||
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||||
|
Prepared: prepared,
|
||||||
|
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||||
|
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkIndexes := extractor.chunkIndexesSnapshot()
|
||||||
|
sort.Ints(chunkIndexes)
|
||||||
|
if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) {
|
||||||
|
t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes)
|
||||||
|
}
|
||||||
|
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||||
|
t.Fatalf("run output = %#v, want approved normalized output without rejections", output)
|
||||||
|
}
|
||||||
|
if output.NormalizeOutputs[0].NormalizerKey != spellnormalize.Key {
|
||||||
|
t.Fatalf("normalized output module = %q, want %q", output.NormalizeOutputs[0].NormalizerKey, spellnormalize.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalized dnd.SpellList
|
||||||
|
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
|
||||||
|
t.Fatalf("decode normalized output: %v", err)
|
||||||
|
}
|
||||||
|
if len(normalized.SpellCasts) != 2 {
|
||||||
|
t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts)
|
||||||
|
}
|
||||||
|
first, distinct := normalized.SpellCasts[0], normalized.SpellCasts[1]
|
||||||
|
if first.Spell != "Cure Wounds" || first.Caster != " Aria \t" || first.Effect != "first occurrence" || first.NarrativeDescription != "first narrative" {
|
||||||
|
t.Fatalf("retained cast = %#v, want canonical spell with first occurrence fields", first)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(first.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||||
|
t.Fatalf("retained refs = %#v, want sorted complete evidence", first.SourceRefs)
|
||||||
|
}
|
||||||
|
if distinct.Spell != "Cure Wounds" || distinct.Caster != "aria" || !reflect.DeepEqual(distinct.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||||
|
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantWarningReasons := []string{
|
||||||
|
spellnormalize.ReasonCodeSpellNameCanonicalized,
|
||||||
|
spellnormalize.ReasonCodeSourceReferencesNormalized,
|
||||||
|
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
|
||||||
|
"spell_not_near_source",
|
||||||
|
}
|
||||||
|
gotWarningReasons := make([]string, len(output.Warnings))
|
||||||
|
for index, warning := range output.Warnings {
|
||||||
|
gotWarningReasons[index] = warning.ReasonCode
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
|
||||||
|
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
|
||||||
|
}
|
||||||
|
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
|
||||||
|
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
|
||||||
|
}
|
||||||
|
|
||||||
|
warningsFile := decodeAssembledOutput[struct {
|
||||||
|
Warnings []contracts.Warning `json:"warnings"`
|
||||||
|
}](t, output.OutputFiles, "warnings.json")
|
||||||
|
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
||||||
|
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
|
||||||
|
}
|
||||||
|
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
||||||
|
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
||||||
|
t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes)
|
||||||
|
}
|
||||||
|
normalizerMetadata, ok := manifest.ArtifactLanes[0].Metadata["normalizer"].(map[string]any)
|
||||||
|
_, hasOverlayIDs := normalizerMetadata["catalog_overlay_ids"]
|
||||||
|
if !ok || normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !hasOverlayIDs {
|
||||||
|
t.Fatalf("normalizer manifest metadata = %#v, want base ID, digest, and overlay IDs", manifest.ArtifactLanes[0].Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||||
|
registries, resolved, _ := assembledSpellPipeline(t, true)
|
||||||
|
var normalizeChain *pipeline.ResolvedValidatorChain
|
||||||
|
for index := range resolved.ValidatorChains {
|
||||||
|
chain := &resolved.ValidatorChains[index]
|
||||||
|
if chain.Stage == pipeline.StageNormalize && chain.ModuleKey == spellnormalize.Key && chain.LaneID == "spells" {
|
||||||
|
normalizeChain = chain
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if normalizeChain == nil || len(normalizeChain.Validators) != 1 || normalizeChain.Validators[0].Binding.Module != "generic/always_accept" {
|
||||||
|
t.Fatalf("normalize validator chain = %#v, want explicit always-accept override", normalizeChain)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||||
|
Prepared: prepared,
|
||||||
|
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||||
|
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||||
|
})
|
||||||
|
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||||
|
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
||||||
|
}
|
||||||
|
for _, warning := range output.Warnings {
|
||||||
|
if warning.ReasonCode == "spell_not_near_source" {
|
||||||
|
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assembledSpellPipeline(t *testing.T, override bool) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||||
|
t.Helper()
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
extractor := &assembledSpellExtractor{}
|
||||||
|
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||||
|
Key: assembledSpellExtractorKey,
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks", "source.transcript"},
|
||||||
|
Provides: []string{"dnd.spell_casts"},
|
||||||
|
ArtifactKind: dnd.SpellListKind,
|
||||||
|
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||||
|
return extractor, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register assembled extractor: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize := pipeline.Binding(spellnormalize.Key)
|
||||||
|
if override {
|
||||||
|
normalize.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||||
|
ID: "assembled-dnd-spells",
|
||||||
|
Input: pipeline.Binding("seriatim"),
|
||||||
|
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"spells": {Extract: pipeline.Binding(assembledSpellExtractorKey), Normalize: normalize},
|
||||||
|
},
|
||||||
|
Output: pipeline.Binding("json"),
|
||||||
|
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return components.registries, resolved, extractor
|
||||||
|
}
|
||||||
|
|
||||||
|
type assembledSpellExtractor struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
chunkIndexes []int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
||||||
|
|
||||||
|
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
|
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||||
|
}
|
||||||
|
if req.Chunk == nil || req.Source == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("assembled extractor requires source and chunk")
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
e.chunkIndexes = append(e.chunkIndexes, req.Chunk.Index)
|
||||||
|
e.mu.Unlock()
|
||||||
|
refOne := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}
|
||||||
|
refTwo := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 2, EndUnitID: 2}
|
||||||
|
switch req.Chunk.Index {
|
||||||
|
case 0:
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||||
|
Caster: " Aria \t", Spell: " cure wounds ", Effect: "first occurrence", NarrativeDescription: "first narrative", SourceRefs: []source.SourceRef{refTwo, refOne},
|
||||||
|
}}}}, nil
|
||||||
|
case 1:
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Caster: "aria", Spell: "Cure Wounds", Effect: "removed occurrence", NarrativeDescription: "removed narrative", SourceRefs: []source.SourceRef{refOne, refTwo}},
|
||||||
|
{Caster: "aria", Spell: "Cure Wounds", Effect: "different evidence", NarrativeDescription: "different narrative", SourceRefs: []source.SourceRef{refTwo}},
|
||||||
|
}}}, nil
|
||||||
|
default:
|
||||||
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("unexpected assembled chunk index %d", req.Chunk.Index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
return append([]int(nil), e.chunkIndexes...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
||||||
|
t.Helper()
|
||||||
|
for _, file := range files {
|
||||||
|
if file.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var value T
|
||||||
|
if err := json.Unmarshal(file.Bytes, &value); err != nil {
|
||||||
|
t.Fatalf("decode %s: %v", name, err)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
t.Fatalf("output files = %#v, want %q", files, name)
|
||||||
|
return *new(T)
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||||
@@ -31,7 +32,9 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|||||||
t.Fatalf("materialize maintained example references: %v", err)
|
t.Fatalf("materialize maintained example references: %v", err)
|
||||||
}
|
}
|
||||||
if example.name == "production" {
|
if example.name == "production" {
|
||||||
if len(materialized.ArtifactLanes) != 1 || len(materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
|
if len(materialized.ArtifactLanes) != 1 ||
|
||||||
|
len(materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
|
||||||
|
len(materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
|
||||||
t.Fatalf("production spell catalog reference was not materialized: %#v", materialized.ArtifactLanes)
|
t.Fatalf("production spell catalog reference was not materialized: %#v", materialized.ArtifactLanes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -80,7 +83,7 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|||||||
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
||||||
}
|
}
|
||||||
lane := manifest.ArtifactLanes[0]
|
lane := manifest.ArtifactLanes[0]
|
||||||
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != "noop" {
|
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != spellnormalize.Key {
|
||||||
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
||||||
}
|
}
|
||||||
if len(manifest.References) != 0 {
|
if len(manifest.References) != 0 {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||||
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -35,7 +36,7 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
|
assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
|
||||||
assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
||||||
assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
|
assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
|
||||||
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
|
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop", spellnormalize.Key})
|
||||||
assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
||||||
assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||||
"extract/dnd/spells/catalog",
|
"extract/dnd/spells/catalog",
|
||||||
@@ -50,6 +51,7 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind})
|
assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind})
|
assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind})
|
assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
|
assertProductionContains(t, "spell normalizer variants", registries.Normalizers.RegisteredArtifactKinds(spellnormalize.Key), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
|
|
||||||
wantChain := []pipeline.ModuleBinding{
|
wantChain := []pipeline.ModuleBinding{
|
||||||
pipeline.Binding("generic/valid_json"),
|
pipeline.Binding("generic/valid_json"),
|
||||||
@@ -62,6 +64,9 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
||||||
}
|
}
|
||||||
|
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
|
t.Fatalf("spell normalize validator chain = %#v, want %#v", got, wantChain)
|
||||||
|
}
|
||||||
|
|
||||||
assetNames := productionAssetNames(t, components.assets.PromptFS)
|
assetNames := productionAssetNames(t, components.assets.PromptFS)
|
||||||
requiredAssets := []string{
|
requiredAssets := []string{
|
||||||
@@ -93,6 +98,9 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
|
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
|
||||||
}
|
}
|
||||||
|
if got := catalog.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
|
t.Fatalf("catalog spell normalize validator chain = %#v, want %#v", got, wantChain)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,9 +148,13 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("materialize production spell references: %v", err)
|
t.Fatalf("materialize production spell references: %v", err)
|
||||||
}
|
}
|
||||||
items := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items
|
extractItems := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items
|
||||||
if len(items) != 1 || items[0].MediaType != "application/json" || len(items[0].Content) == 0 {
|
normalizeItems := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items
|
||||||
t.Fatalf("materialized spell catalog items = %#v, want one JSON item", items)
|
if len(extractItems) != 1 || extractItems[0].MediaType != "application/json" || len(extractItems[0].Content) == 0 {
|
||||||
|
t.Fatalf("materialized extract spell catalog items = %#v, want one JSON item", extractItems)
|
||||||
|
}
|
||||||
|
if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) {
|
||||||
|
t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems)
|
||||||
}
|
}
|
||||||
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||||
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
|
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
|
||||||
@@ -235,7 +247,7 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
|
|||||||
name: "invalid artifact variant",
|
name: "invalid artifact variant",
|
||||||
content: base,
|
content: base,
|
||||||
options: productionCLIOptionsWithoutSpellNormalizer(t),
|
options: productionCLIOptionsWithoutSpellNormalizer(t),
|
||||||
fragments: []string{"normalizer", "noop", string(dnd.SpellListKind), "variant"},
|
fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "deterministic validator with profile",
|
name: "deterministic validator with profile",
|
||||||
@@ -261,6 +273,27 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
|
||||||
|
base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
|
||||||
|
content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n")
|
||||||
|
path := writeProductionContractConfig(t, content)
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve normalize override: %v", err)
|
||||||
|
}
|
||||||
|
for _, chain := range effective.ResolvedPipeline.ValidatorChains {
|
||||||
|
if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(chain.Validators) != 1 || chain.Validators[0].Binding.Module != "generic/always_accept" {
|
||||||
|
t.Fatalf("normalize validator chain = %#v, want explicit always-accept override", chain)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key)
|
||||||
|
}
|
||||||
|
|
||||||
func TestProductionSceneRunRecordsChunkerWarningsAndProvenance(t *testing.T) {
|
func TestProductionSceneRunRecordsChunkerWarningsAndProvenance(t *testing.T) {
|
||||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
|
configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
|
||||||
|
|||||||
@@ -14,9 +14,11 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -41,6 +43,18 @@ func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testin
|
|||||||
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
||||||
}
|
}
|
||||||
resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
||||||
|
normalizeBindings := resolved.ArtifactLanes[0].NormalizeReferences.Bindings
|
||||||
|
normalizeCatalogBindingIndex := -1
|
||||||
|
for index, binding := range normalizeBindings {
|
||||||
|
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||||
|
normalizeCatalogBindingIndex = index
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if normalizeCatalogBindingIndex < 0 {
|
||||||
|
t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
|
||||||
|
}
|
||||||
|
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath
|
||||||
|
|
||||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -51,6 +65,7 @@ func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testin
|
|||||||
}
|
}
|
||||||
identityA := catalogCheckpointIdentity(t, materializedA)
|
identityA := catalogCheckpointIdentity(t, materializedA)
|
||||||
metadataA := catalogExtractorMetadata(t, materializedA)
|
metadataA := catalogExtractorMetadata(t, materializedA)
|
||||||
|
normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
|
||||||
referenceA := catalogReference(t, materializedA)
|
referenceA := catalogReference(t, materializedA)
|
||||||
|
|
||||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
||||||
@@ -62,6 +77,7 @@ func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testin
|
|||||||
}
|
}
|
||||||
identityB := catalogCheckpointIdentity(t, materializedB)
|
identityB := catalogCheckpointIdentity(t, materializedB)
|
||||||
metadataB := catalogExtractorMetadata(t, materializedB)
|
metadataB := catalogExtractorMetadata(t, materializedB)
|
||||||
|
normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
|
||||||
referenceB := catalogReference(t, materializedB)
|
referenceB := catalogReference(t, materializedB)
|
||||||
|
|
||||||
if identityA.Digest == identityB.Digest {
|
if identityA.Digest == identityB.Digest {
|
||||||
@@ -81,6 +97,14 @@ func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testin
|
|||||||
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
||||||
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
||||||
}
|
}
|
||||||
|
normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
|
||||||
|
normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
|
||||||
|
if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
|
||||||
|
t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
|
||||||
|
}
|
||||||
|
if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
|
||||||
|
t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
||||||
@@ -128,8 +152,28 @@ func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenc
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
fingerprints := prepared.CheckpointFingerprints()
|
fingerprints := prepared.CheckpointFingerprints()
|
||||||
if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value {
|
wantNames := map[string]struct{}{
|
||||||
t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints)
|
"extract:spells:" + spells.Key + ":effective_catalog": {},
|
||||||
|
"extract:spells:" + spells.Key + ":validator:3:extract/dnd/spells/catalog:effective_catalog": {},
|
||||||
|
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
|
||||||
|
"normalize:spells:" + spellnormalize.Key + ":validator:3:extract/dnd/spells/catalog:effective_catalog": {},
|
||||||
|
}
|
||||||
|
seen := make(map[string]string, len(fingerprints))
|
||||||
|
for _, fingerprint := range fingerprints {
|
||||||
|
if _, ok := wantNames[fingerprint.Name]; ok {
|
||||||
|
seen[fingerprint.Name] = fingerprint.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(seen) != len(wantNames) {
|
||||||
|
t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
|
||||||
|
}
|
||||||
|
var catalogDigest string
|
||||||
|
for name, value := range seen {
|
||||||
|
if catalogDigest == "" {
|
||||||
|
catalogDigest = value
|
||||||
|
} else if value != catalogDigest {
|
||||||
|
t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
||||||
@@ -184,6 +228,18 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
|
||||||
|
normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
|
||||||
|
normalizeArtifact := pipeline.CheckpointArtifact{
|
||||||
|
LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
|
||||||
|
SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
|
||||||
|
Artifact: contracts.SerializedArtifact{
|
||||||
|
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
|
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -192,6 +248,9 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
||||||
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
||||||
}
|
}
|
||||||
|
if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
|
||||||
|
t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
|
||||||
|
}
|
||||||
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
||||||
changed[0].Value = "sha256:changed-effective-catalog"
|
changed[0].Value = "sha256:changed-effective-catalog"
|
||||||
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
|
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
|
||||||
@@ -201,6 +260,9 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
||||||
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
|
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||||
|
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
||||||
@@ -234,21 +296,35 @@ func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t
|
|||||||
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
||||||
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
||||||
}
|
}
|
||||||
|
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
|
||||||
|
}
|
||||||
|
if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
|
||||||
|
t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
|
||||||
|
}
|
||||||
|
if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
|
||||||
|
t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
var catalogProvenance *artifacts.ReferenceProvenance
|
var catalogProvenances []artifacts.ReferenceProvenance
|
||||||
for index := range manifest.References {
|
for index := range manifest.References {
|
||||||
reference := &manifest.References[index]
|
reference := &manifest.References[index]
|
||||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||||
catalogProvenance = reference
|
catalogProvenances = append(catalogProvenances, *reference)
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if catalogProvenance == nil {
|
if len(catalogProvenances) != 2 {
|
||||||
t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References)
|
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
|
||||||
}
|
}
|
||||||
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
|
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
|
||||||
if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
|
for _, catalogProvenance := range catalogProvenances {
|
||||||
t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance)
|
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
|
||||||
|
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
|
||||||
|
}
|
||||||
|
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
|
||||||
|
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
manifestBytes, err := json.Marshal(manifest)
|
manifestBytes, err := json.Marshal(manifest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -304,6 +380,16 @@ func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline)
|
|||||||
return extractor.ManifestMetadata()
|
return extractor.ManifestMetadata()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
lane := resolved.ArtifactLanes[0]
|
||||||
|
normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct normalizer: %v", err)
|
||||||
|
}
|
||||||
|
return normalizer.ManifestMetadata()
|
||||||
|
}
|
||||||
|
|
||||||
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
||||||
|
|||||||
@@ -106,8 +106,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
|||||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||||
}
|
}
|
||||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != tt.wantWarningCode {
|
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode {
|
||||||
t.Fatalf("warnings = %#v, want only accepted-attempt warning", output.Warnings)
|
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
56
internal/modules/dnd/normalize/spells/fixture_test.go
Normal file
56
internal/modules/dnd/normalize/spells/fixture_test.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package spells
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
type normalizerFixtureSet struct {
|
||||||
|
Cases []normalizerFixtureCase `json:"cases"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type normalizerFixtureCase struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Input dnd.SpellList `json:"input"`
|
||||||
|
Output dnd.SpellList `json:"output"`
|
||||||
|
WarningReasonCodes []string `json:"warning_reason_codes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeAcceptedFixtures(t *testing.T) {
|
||||||
|
data, err := os.ReadFile("testdata/normalizer_accepted_cases.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture: %v", err)
|
||||||
|
}
|
||||||
|
var fixtures normalizerFixtureSet
|
||||||
|
if err := json.Unmarshal(data, &fixtures); err != nil {
|
||||||
|
t.Fatalf("decode fixture: %v", err)
|
||||||
|
}
|
||||||
|
if len(fixtures.Cases) == 0 {
|
||||||
|
t.Fatal("fixture set is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, fixture := range fixtures.Cases {
|
||||||
|
t.Run(fixture.Name, func(t *testing.T) {
|
||||||
|
result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(fixture.Input, sourceDocument(2)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(result.Value, fixture.Output) {
|
||||||
|
t.Fatalf("normalized value = %#v, want %#v", result.Value, fixture.Output)
|
||||||
|
}
|
||||||
|
|
||||||
|
gotReasonCodes := make([]string, 0, len(result.Warnings))
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
gotReasonCodes = append(gotReasonCodes, warning.ReasonCode)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotReasonCodes, fixture.WarningReasonCodes) {
|
||||||
|
t.Fatalf("warning reason codes = %#v, want %#v", gotReasonCodes, fixture.WarningReasonCodes)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
356
internal/modules/dnd/normalize/spells/normalizer.go
Normal file
356
internal/modules/dnd/normalize/spells/normalizer.go
Normal file
@@ -0,0 +1,356 @@
|
|||||||
|
package spells
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
|
"golang.org/x/text/cases"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "dnd/spells"
|
||||||
|
|
||||||
|
const (
|
||||||
|
ReasonCodeSpellNameCanonicalized = "spell_name_canonicalized"
|
||||||
|
ReasonCodeSpellNameUnresolved = "spell_name_unresolved"
|
||||||
|
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
||||||
|
ReasonCodeDuplicateSpellCastCollapsed = "duplicate_spell_cast_collapsed"
|
||||||
|
)
|
||||||
|
|
||||||
|
var requiredCapabilities = []string{"merged"}
|
||||||
|
var providedCapabilities = []string{"normalized"}
|
||||||
|
|
||||||
|
var _ contracts.Normalizer[dnd.SpellList] = (*Normalizer)(nil)
|
||||||
|
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||||
|
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||||
|
|
||||||
|
type Options struct{}
|
||||||
|
|
||||||
|
type Normalizer struct {
|
||||||
|
effectiveCatalog spellcatalog.EffectiveCatalog
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(_ Options, references ...contracts.ReferenceSet) (*Normalizer, error) {
|
||||||
|
if len(references) > 1 {
|
||||||
|
return nil, normalizerErrorf("at most one reference set may be supplied")
|
||||||
|
}
|
||||||
|
var referenceSet contracts.ReferenceSet
|
||||||
|
if len(references) == 1 {
|
||||||
|
referenceSet = references[0]
|
||||||
|
}
|
||||||
|
effectiveCatalog, err := spellcatalog.ResolveEffectiveCatalog(referenceSet)
|
||||||
|
if err != nil {
|
||||||
|
return nil, normalizerErrorf("resolve effective spell catalog: %w", err)
|
||||||
|
}
|
||||||
|
return &Normalizer{effectiveCatalog: effectiveCatalog}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) Key() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return referenceSlots()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||||
|
if n == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"catalog_base_id": n.effectiveCatalog.BaseID(),
|
||||||
|
"catalog_digest": n.effectiveCatalog.Digest(),
|
||||||
|
"catalog_overlay_ids": append([]string(nil), n.effectiveCatalog.OverlayIDs()...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||||
|
if n == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: n.effectiveCatalog.Digest()}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.SpellList]) (contracts.TypedNormalizeResult[dnd.SpellList], error) {
|
||||||
|
if n == nil {
|
||||||
|
return contracts.TypedNormalizeResult[dnd.SpellList]{}, normalizerErrorf("normalizer must not be nil")
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
return contracts.TypedNormalizeResult[dnd.SpellList]{}, normalizerErrorf("context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.TypedNormalizeResult[dnd.SpellList]{}, normalizerErrorf("context error before normalize: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
value, warnings := normalizeSpellList(req.MergeOutput.Value, n.effectiveCatalog)
|
||||||
|
value, duplicateWarnings := collapseDuplicateSpellCasts(value, req.Source, n.effectiveCatalog)
|
||||||
|
warnings = append(warnings, duplicateWarnings...)
|
||||||
|
return contracts.TypedNormalizeResult[dnd.SpellList]{Value: value, Warnings: warnings}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeSpellList(input dnd.SpellList, catalog spellcatalog.EffectiveCatalog) (dnd.SpellList, []contracts.Warning) {
|
||||||
|
var warnings []contracts.Warning
|
||||||
|
if input.SpellCasts == nil {
|
||||||
|
return dnd.SpellList{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
output := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, len(input.SpellCasts))}
|
||||||
|
for index, inputCast := range input.SpellCasts {
|
||||||
|
cast := cloneSpellCast(inputCast)
|
||||||
|
if canonicalName, ok := catalog.Lookup(inputCast.Spell); ok {
|
||||||
|
if inputCast.Spell != canonicalName {
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: spellCastScope(index),
|
||||||
|
ReasonCode: ReasonCodeSpellNameCanonicalized,
|
||||||
|
Message: fmt.Sprintf("input index %d: spell name canonicalized from %q to %q",
|
||||||
|
index, boundedName(inputCast.Spell), boundedName(canonicalName)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
cast.Spell = canonicalName
|
||||||
|
} else {
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: spellCastScope(index),
|
||||||
|
ReasonCode: ReasonCodeSpellNameUnresolved,
|
||||||
|
Message: fmt.Sprintf("input index %d: spell name %q could not be resolved in the effective catalog",
|
||||||
|
index, boundedName(inputCast.Spell)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalRefs, orderChanged, duplicateCount := canonicalizeSourceRefs(inputCast.SourceRefs)
|
||||||
|
cast.SourceRefs = canonicalRefs
|
||||||
|
if orderChanged || duplicateCount > 0 {
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: spellCastScope(index),
|
||||||
|
ReasonCode: ReasonCodeSourceReferencesNormalized,
|
||||||
|
Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d, order changed %t, duplicates removed %d)",
|
||||||
|
index, len(inputCast.SourceRefs), len(canonicalRefs), orderChanged, duplicateCount),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
output.SpellCasts[index] = cast
|
||||||
|
}
|
||||||
|
return output, warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSpellCast(input dnd.SpellCast) dnd.SpellCast {
|
||||||
|
output := input
|
||||||
|
if input.SourceRefs != nil {
|
||||||
|
output.SourceRefs = make([]source.SourceRef, len(input.SourceRefs))
|
||||||
|
copy(output.SourceRefs, input.SourceRefs)
|
||||||
|
}
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalizeSourceRefs(input []source.SourceRef) ([]source.SourceRef, bool, int) {
|
||||||
|
if input == nil {
|
||||||
|
return nil, false, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
canonical := make([]source.SourceRef, len(input))
|
||||||
|
copy(canonical, input)
|
||||||
|
sort.SliceStable(canonical, func(left, right int) bool {
|
||||||
|
return sourceRefLess(canonical[left], canonical[right])
|
||||||
|
})
|
||||||
|
|
||||||
|
orderChanged := false
|
||||||
|
for index := range input {
|
||||||
|
if input[index] != canonical[index] {
|
||||||
|
orderChanged = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unique := make([]source.SourceRef, 0, len(canonical))
|
||||||
|
for _, ref := range canonical {
|
||||||
|
if len(unique) == 0 || unique[len(unique)-1] != ref {
|
||||||
|
unique = append(unique, ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unique, orderChanged, len(input) - len(unique)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceRefLess(left, right source.SourceRef) bool {
|
||||||
|
if left.SourceID != right.SourceID {
|
||||||
|
return left.SourceID < right.SourceID
|
||||||
|
}
|
||||||
|
if left.StartUnitID != right.StartUnitID {
|
||||||
|
return left.StartUnitID < right.StartUnitID
|
||||||
|
}
|
||||||
|
return left.EndUnitID < right.EndUnitID
|
||||||
|
}
|
||||||
|
|
||||||
|
type duplicateGroup struct {
|
||||||
|
retainedIndex int
|
||||||
|
removed []int
|
||||||
|
}
|
||||||
|
|
||||||
|
func collapseDuplicateSpellCasts(input dnd.SpellList, doc *source.SourceDocument, catalog spellcatalog.EffectiveCatalog) (dnd.SpellList, []contracts.Warning) {
|
||||||
|
if len(input.SpellCasts) == 0 {
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keep := make([]bool, len(input.SpellCasts))
|
||||||
|
groups := make([]duplicateGroup, 0)
|
||||||
|
groupByKey := make(map[string]int)
|
||||||
|
for index, cast := range input.SpellCasts {
|
||||||
|
key, eligible := duplicateKey(cast, doc, catalog)
|
||||||
|
if !eligible {
|
||||||
|
keep[index] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
groupIndex, exists := groupByKey[key]
|
||||||
|
if !exists {
|
||||||
|
groupByKey[key] = len(groups)
|
||||||
|
groups = append(groups, duplicateGroup{retainedIndex: index})
|
||||||
|
keep[index] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
groups[groupIndex].removed = append(groups[groupIndex].removed, index)
|
||||||
|
}
|
||||||
|
|
||||||
|
removedAny := false
|
||||||
|
for _, group := range groups {
|
||||||
|
if len(group.removed) > 0 {
|
||||||
|
removedAny = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !removedAny {
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
output := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, 0, len(input.SpellCasts))}
|
||||||
|
for index, cast := range input.SpellCasts {
|
||||||
|
if keep[index] {
|
||||||
|
output.SpellCasts = append(output.SpellCasts, cast)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
warnings := make([]contracts.Warning, 0)
|
||||||
|
for _, group := range groups {
|
||||||
|
if len(group.removed) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
warnings = append(warnings, duplicateWarning(group.retainedIndex, group.removed))
|
||||||
|
}
|
||||||
|
return output, warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func duplicateKey(cast dnd.SpellCast, doc *source.SourceDocument, catalog spellcatalog.EffectiveCatalog) (string, bool) {
|
||||||
|
canonicalName, resolved := catalog.Lookup(cast.Spell)
|
||||||
|
if !resolved || len(cast.SourceRefs) == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
for _, ref := range cast.SourceRefs {
|
||||||
|
if source.ValidateRef(doc, ref) != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var key strings.Builder
|
||||||
|
writeKeyString(&key, canonicalName)
|
||||||
|
writeKeyString(&key, cases.Fold().String(strings.Join(strings.Fields(cast.Caster), " ")))
|
||||||
|
for _, ref := range cast.SourceRefs {
|
||||||
|
writeKeyString(&key, ref.SourceID)
|
||||||
|
writeKeyInt(&key, ref.StartUnitID)
|
||||||
|
writeKeyInt(&key, ref.EndUnitID)
|
||||||
|
}
|
||||||
|
return key.String(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeKeyString(builder *strings.Builder, value string) {
|
||||||
|
builder.WriteString(strconv.Itoa(len(value)))
|
||||||
|
builder.WriteByte(':')
|
||||||
|
builder.WriteString(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeKeyInt(builder *strings.Builder, value int) {
|
||||||
|
builder.WriteString(strconv.Itoa(value))
|
||||||
|
builder.WriteByte(';')
|
||||||
|
}
|
||||||
|
|
||||||
|
func duplicateWarning(retainedIndex int, removed []int) contracts.Warning {
|
||||||
|
const maxDisplayedIndices = 20
|
||||||
|
displayed := removed
|
||||||
|
if len(displayed) > maxDisplayedIndices {
|
||||||
|
displayed = displayed[:maxDisplayedIndices]
|
||||||
|
}
|
||||||
|
indices := make([]string, len(displayed))
|
||||||
|
for index, removedIndex := range displayed {
|
||||||
|
indices[index] = strconv.Itoa(removedIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", "))
|
||||||
|
if omitted := len(removed) - len(displayed); omitted > 0 {
|
||||||
|
message += fmt.Sprintf("; %d additional removed input indices omitted", omitted)
|
||||||
|
}
|
||||||
|
return contracts.Warning{
|
||||||
|
Scope: spellCastScope(retainedIndex),
|
||||||
|
ReasonCode: ReasonCodeDuplicateSpellCastCollapsed,
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boundedName(name string) string {
|
||||||
|
runes := []rune(name)
|
||||||
|
if len(runes) <= 128 {
|
||||||
|
return string(runes)
|
||||||
|
}
|
||||||
|
return string(runes[:127]) + "…"
|
||||||
|
}
|
||||||
|
|
||||||
|
func spellCastScope(index int) string {
|
||||||
|
return fmt.Sprintf("spell_casts[%d]", index)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ModuleSpec() pipeline.ModuleSpec {
|
||||||
|
return pipeline.ModuleSpec{
|
||||||
|
Key: Key,
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: append([]string(nil), requiredCapabilities...),
|
||||||
|
Provides: append([]string(nil), providedCapabilities...),
|
||||||
|
ArtifactKind: dnd.SpellListKind,
|
||||||
|
ReferenceSlots: referenceSlots(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||||
|
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.SpellList], error) {
|
||||||
|
options, err := DecodeOptions(request.Options)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return New(options, request.References)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOptions(options map[string]any) error {
|
||||||
|
_, err := DecodeOptions(options)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func DecodeOptions(options map[string]any) (Options, error) {
|
||||||
|
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||||
|
return Options{}, normalizerErrorf("%w", err)
|
||||||
|
}
|
||||||
|
return Options{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return []contracts.ReferenceSlot{{
|
||||||
|
Name: spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
Description: "Optional canonical spell-name catalog used for extraction grounding.",
|
||||||
|
AcceptedMediaTypes: []string{"application/json"},
|
||||||
|
MaxBytes: 1048576,
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizerErrorf(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("dnd spells normalizer: "+format, args...)
|
||||||
|
}
|
||||||
482
internal/modules/dnd/normalize/spells/normalizer_test.go
Normal file
482
internal/modules/dnd/normalize/spells/normalizer_test.go
Normal file
@@ -0,0 +1,482 @@
|
|||||||
|
package spells
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestModuleContractAndStrictOptions(t *testing.T) {
|
||||||
|
if _, err := DecodeOptions(nil); err != nil {
|
||||||
|
t.Fatalf("DecodeOptions(nil) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||||
|
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := pipeline.ModuleSpec{
|
||||||
|
Key: Key,
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ArtifactKind: dnd.SpellListKind,
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{
|
||||||
|
Name: spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
Description: "Optional canonical spell-name catalog used for extraction grounding.",
|
||||||
|
AcceptedMediaTypes: []string{"application/json"},
|
||||||
|
MaxBytes: 1048576,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewNormalizerRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
registered, ok := registry.SpecForArtifact(Key, dnd.SpellListKind)
|
||||||
|
if !ok || !reflect.DeepEqual(registered, want) {
|
||||||
|
t.Fatalf("registered spec = %#v, ok = %t, want %#v", registered, ok, want)
|
||||||
|
}
|
||||||
|
if err := Register(nil); err == nil || !strings.Contains(err.Error(), "normalizer registry") {
|
||||||
|
t.Fatalf("Register(nil) error = %v, want registry error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewBuildsEmbeddedAndOverlayCatalogs(t *testing.T) {
|
||||||
|
base, err := New(Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if base.effectiveCatalog.BaseID() != spellcatalog.SRD5E2014ID || base.effectiveCatalog.Digest() == "" || len(base.effectiveCatalog.OverlayIDs()) != 0 {
|
||||||
|
t.Fatalf("base catalog identity = %#v, want embedded catalog identity", base.effectiveCatalog)
|
||||||
|
}
|
||||||
|
if canonical, ok := base.effectiveCatalog.Lookup(" cure wounds "); !ok || canonical != "Cure Wounds" {
|
||||||
|
t.Fatalf("base catalog lookup = %q, %t, want Cure Wounds", canonical, ok)
|
||||||
|
}
|
||||||
|
if canonical, ok := base.effectiveCatalog.Lookup("Arcanist's Magic Aura"); !ok || canonical != "Arcanist’s Magic Aura" {
|
||||||
|
t.Fatalf("apostrophe lookup = %q, %t, want canonical curly apostrophe spelling", canonical, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
overlay, err := New(Options{}, overlayReference())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(overlay) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := overlay.effectiveCatalog.OverlayIDs(); !reflect.DeepEqual(got, []string{"campaign.example"}) {
|
||||||
|
t.Fatalf("overlay IDs = %#v, want campaign.example", got)
|
||||||
|
}
|
||||||
|
if canonical, ok := overlay.effectiveCatalog.Lookup(" emberfall aegis "); !ok || canonical != "Aegis of Emberfall" {
|
||||||
|
t.Fatalf("overlay alias lookup = %q, %t, want Aegis of Emberfall", canonical, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRejectsInvalidCatalogReferencesDuringConstruction(t *testing.T) {
|
||||||
|
invalid := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
spellcatalog.SpellCatalogReferenceSlot: {
|
||||||
|
Items: []contracts.ReferenceItem{{MediaType: "text/plain", Content: []byte("not a catalog")}},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
if _, err := New(Options{}, invalid); err == nil || !strings.Contains(err.Error(), "application/json") {
|
||||||
|
t.Fatalf("New(invalid reference) error = %v, want media type failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tooMany := overlayReference()
|
||||||
|
slot := tooMany.Slots[spellcatalog.SpellCatalogReferenceSlot]
|
||||||
|
slot.Items = append(slot.Items, slot.Items[0])
|
||||||
|
tooMany.Slots[spellcatalog.SpellCatalogReferenceSlot] = slot
|
||||||
|
if _, err := New(Options{}, tooMany); err == nil || !strings.Contains(err.Error(), "zero or one item") {
|
||||||
|
t.Fatalf("New(duplicated reference) error = %v, want multiplicity failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentityAndMetadataAreDefensive(t *testing.T) {
|
||||||
|
normalizer, err := New(Options{}, overlayReference())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fingerprints := normalizer.CheckpointFingerprints()
|
||||||
|
if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != normalizer.effectiveCatalog.Digest() {
|
||||||
|
t.Fatalf("fingerprints = %#v, want effective catalog fingerprint", fingerprints)
|
||||||
|
}
|
||||||
|
fingerprints[0].Name = "changed"
|
||||||
|
fingerprints[0].Value = "changed"
|
||||||
|
if got := normalizer.CheckpointFingerprints(); len(got) != 1 || got[0].Name != "effective_catalog" || got[0].Value != normalizer.effectiveCatalog.Digest() {
|
||||||
|
t.Fatalf("fingerprints were not defensive: %#v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata := normalizer.ManifestMetadata()
|
||||||
|
if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || metadata["catalog_digest"] != normalizer.effectiveCatalog.Digest() {
|
||||||
|
t.Fatalf("metadata = %#v, want catalog identity", metadata)
|
||||||
|
}
|
||||||
|
metadata["catalog_overlay_ids"].([]string)[0] = "changed"
|
||||||
|
if got := normalizer.ManifestMetadata()["catalog_overlay_ids"].([]string); !reflect.DeepEqual(got, []string{"campaign.example"}) {
|
||||||
|
t.Fatalf("metadata overlay IDs were not defensive: %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeCanonicalizesNamesAndReportsUnresolvedNames(t *testing.T) {
|
||||||
|
normalizer := newNormalizer(t)
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Spell: " cure wounds "},
|
||||||
|
{Spell: "Arcanist's Magic Aura"},
|
||||||
|
{Spell: "Emberfall Aegis"},
|
||||||
|
{Spell: "Mystery\nSpell\tName"},
|
||||||
|
{Spell: "Cure Wounds"},
|
||||||
|
}}
|
||||||
|
result, err := normalizer.Normalize(context.Background(), normalizeRequest(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
wantNames := []string{"Cure Wounds", "Arcanist’s Magic Aura", "Aegis of Emberfall", "Mystery\nSpell\tName", "Cure Wounds"}
|
||||||
|
for index, want := range wantNames {
|
||||||
|
if result.Value.SpellCasts[index].Spell != want {
|
||||||
|
t.Fatalf("spell[%d] = %q, want %q", index, result.Value.SpellCasts[index].Spell, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 4 {
|
||||||
|
t.Fatalf("warnings = %#v, want four name warnings", result.Warnings)
|
||||||
|
}
|
||||||
|
if result.Warnings[0].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[0].Scope != "spell_casts[0]" || !strings.Contains(result.Warnings[0].Message, "input index 0") {
|
||||||
|
t.Fatalf("first warning = %#v, want canonicalization warning", result.Warnings[0])
|
||||||
|
}
|
||||||
|
if result.Warnings[1].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[2].ReasonCode != ReasonCodeSpellNameCanonicalized {
|
||||||
|
t.Fatalf("catalog spelling warnings = %#v", result.Warnings[1:3])
|
||||||
|
}
|
||||||
|
if result.Warnings[3].ReasonCode != ReasonCodeSpellNameUnresolved || result.Warnings[3].Scope != "spell_casts[3]" || strings.Contains(result.Warnings[3].Message, "Mystery\nSpell") || !strings.Contains(result.Warnings[3].Message, `Mystery\nSpell\tName`) {
|
||||||
|
t.Fatalf("unresolved warning = %#v, want quoted control characters", result.Warnings[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeBoundsUnicodeNamesAndQuotesCanonicalReplacement(t *testing.T) {
|
||||||
|
longName := strings.Repeat("火", 140)
|
||||||
|
reference := spellCatalogReference(fmt.Sprintf(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.long","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source"},"spells":[{"name":%q,"aliases":["long alias"]}]}]}`, longName))
|
||||||
|
normalizer, err := New(Options{}, reference)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
result, err := normalizer.Normalize(context.Background(), normalizeRequest(dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "long alias"}}}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != ReasonCodeSpellNameCanonicalized {
|
||||||
|
t.Fatalf("warnings = %#v, want one canonicalization warning", result.Warnings)
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(result.Warnings[0].Message) || !strings.Contains(result.Warnings[0].Message, "…") || strings.Contains(result.Warnings[0].Message, longName) {
|
||||||
|
t.Fatalf("warning = %q, want valid bounded Unicode diagnostic", result.Warnings[0].Message)
|
||||||
|
}
|
||||||
|
if got := result.Value.SpellCasts[0].Spell; got != longName {
|
||||||
|
t.Fatalf("canonical value = %q, want full catalog name", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSortsAndDeduplicatesExactSourceReferences(t *testing.T) {
|
||||||
|
normalizer := newNormalizer(t)
|
||||||
|
inputRefs := []source.SourceRef{
|
||||||
|
{SourceID: "source-b", StartUnitID: 4, EndUnitID: 5},
|
||||||
|
{SourceID: "source-a", StartUnitID: 3, EndUnitID: 4},
|
||||||
|
{SourceID: "source-a", StartUnitID: 3, EndUnitID: 4},
|
||||||
|
{SourceID: "source-a", StartUnitID: 1, EndUnitID: 2},
|
||||||
|
{SourceID: "source-a", StartUnitID: 2, EndUnitID: 3},
|
||||||
|
{SourceID: "source-a", StartUnitID: 1, EndUnitID: 4},
|
||||||
|
}
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "Cure Wounds", SourceRefs: inputRefs}}}
|
||||||
|
result, err := normalizer.Normalize(context.Background(), normalizeRequest(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
wantRefs := []source.SourceRef{
|
||||||
|
{SourceID: "source-a", StartUnitID: 1, EndUnitID: 2},
|
||||||
|
{SourceID: "source-a", StartUnitID: 1, EndUnitID: 4},
|
||||||
|
{SourceID: "source-a", StartUnitID: 2, EndUnitID: 3},
|
||||||
|
{SourceID: "source-a", StartUnitID: 3, EndUnitID: 4},
|
||||||
|
{SourceID: "source-b", StartUnitID: 4, EndUnitID: 5},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(result.Value.SpellCasts[0].SourceRefs, wantRefs) {
|
||||||
|
t.Fatalf("source refs = %#v, want %#v", result.Value.SpellCasts[0].SourceRefs, wantRefs)
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != ReasonCodeSourceReferencesNormalized || !strings.Contains(result.Warnings[0].Message, "original count 6") || !strings.Contains(result.Warnings[0].Message, "final count 5") || !strings.Contains(result.Warnings[0].Message, "duplicates removed 1") {
|
||||||
|
t.Fatalf("warnings = %#v, want source normalization warning", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizePreservesNilEmptyAndAdjacentOrOverlappingReferences(t *testing.T) {
|
||||||
|
normalizer := newNormalizer(t)
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Spell: "Cure Wounds", SourceRefs: nil},
|
||||||
|
{Spell: "Cure Wounds", SourceRefs: []source.SourceRef{}},
|
||||||
|
{Spell: "Cure Wounds", SourceRefs: []source.SourceRef{
|
||||||
|
{SourceID: "source", StartUnitID: 3, EndUnitID: 4},
|
||||||
|
{SourceID: "source", StartUnitID: 1, EndUnitID: 2},
|
||||||
|
{SourceID: "source", StartUnitID: 2, EndUnitID: 5},
|
||||||
|
}},
|
||||||
|
}}
|
||||||
|
before := input
|
||||||
|
before.SpellCasts = append([]dnd.SpellCast(nil), input.SpellCasts...)
|
||||||
|
before.SpellCasts[2].SourceRefs = append([]source.SourceRef(nil), input.SpellCasts[2].SourceRefs...)
|
||||||
|
result, err := normalizer.Normalize(context.Background(), normalizeRequest(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Value.SpellCasts[0].SourceRefs != nil || result.Value.SpellCasts[1].SourceRefs == nil {
|
||||||
|
t.Fatalf("nil/empty source refs were not preserved: %#v", result.Value.SpellCasts)
|
||||||
|
}
|
||||||
|
if len(result.Value.SpellCasts[2].SourceRefs) != 3 {
|
||||||
|
t.Fatalf("adjacent/overlapping references = %#v, want all three retained", result.Value.SpellCasts[2].SourceRefs)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(input, before) {
|
||||||
|
t.Fatalf("Normalize() mutated input: got %#v, before %#v", input, before)
|
||||||
|
}
|
||||||
|
result.Value.SpellCasts[2].SourceRefs[0].SourceID = "changed"
|
||||||
|
if input.SpellCasts[2].SourceRefs[0].SourceID == "changed" {
|
||||||
|
t.Fatal("normalized references share input storage")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) {
|
||||||
|
request := normalizeRequest(dnd.SpellList{})
|
||||||
|
var normalizer *Normalizer
|
||||||
|
if _, err := normalizer.Normalize(context.Background(), request); err == nil || !strings.Contains(err.Error(), "normalizer must not be nil") {
|
||||||
|
t.Fatalf("nil receiver error = %v, want nil receiver error", err)
|
||||||
|
}
|
||||||
|
normalizer = newNormalizer(t)
|
||||||
|
if _, err := normalizer.Normalize(nil, request); err == nil || !strings.Contains(err.Error(), "context must not be nil") {
|
||||||
|
t.Fatalf("nil context error = %v, want nil context error", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := normalizer.Normalize(ctx, request); err == nil || !strings.Contains(err.Error(), "context error before normalize") {
|
||||||
|
t.Fatalf("canceled context error = %v, want canceled context error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeCollapsesDuplicateGroupsAfterCanonicalization(t *testing.T) {
|
||||||
|
normalizer := newNormalizer(t)
|
||||||
|
doc := sourceDocument(6)
|
||||||
|
firstEvidence := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
|
||||||
|
secondEvidence := source.SourceRef{SourceID: "source", StartUnitID: 3, EndUnitID: 4}
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Caster: " Aria \t", Spell: " cure wounds ", Effect: "first effect", NarrativeDescription: "first narrative", SourceRefs: []source.SourceRef{secondEvidence, firstEvidence, firstEvidence}},
|
||||||
|
{Caster: "Borin", Spell: "Healing Word", Effect: "distinct effect", NarrativeDescription: "distinct narrative", SourceRefs: []source.SourceRef{firstEvidence}},
|
||||||
|
{Caster: " Kyle ", Spell: "Cure Wounds", Effect: "kept effect", NarrativeDescription: "kept narrative", SourceRefs: []source.SourceRef{firstEvidence}},
|
||||||
|
{Caster: "aria", Spell: " cure wounds ", Effect: "removed effect", NarrativeDescription: "removed narrative", SourceRefs: []source.SourceRef{firstEvidence, secondEvidence}},
|
||||||
|
{Caster: "KYLE", Spell: " cure wounds ", Effect: "removed effect two", NarrativeDescription: "removed narrative two", SourceRefs: []source.SourceRef{firstEvidence}},
|
||||||
|
{Caster: " kyle ", Spell: "Cure Wounds", Effect: "removed effect three", NarrativeDescription: "removed narrative three", SourceRefs: []source.SourceRef{firstEvidence}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(result.Value.SpellCasts) != 3 {
|
||||||
|
t.Fatalf("normalized casts = %#v, want first occurrences plus distinct cast", result.Value.SpellCasts)
|
||||||
|
}
|
||||||
|
if got := result.Value.SpellCasts[0]; got.Caster != " Aria \t" || got.Effect != "first effect" || got.NarrativeDescription != "first narrative" {
|
||||||
|
t.Fatalf("retained first cast = %#v, want first occurrence fields unchanged", got)
|
||||||
|
}
|
||||||
|
if got := result.Value.SpellCasts[0].SourceRefs; !reflect.DeepEqual(got, []source.SourceRef{firstEvidence, secondEvidence}) {
|
||||||
|
t.Fatalf("retained first evidence = %#v, want canonical first evidence only", got)
|
||||||
|
}
|
||||||
|
if got := result.Value.SpellCasts[2].Caster; got != " Kyle " {
|
||||||
|
t.Fatalf("Unicode caster output = %q, want first occurrence text unchanged", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Warnings) != 6 {
|
||||||
|
t.Fatalf("warnings = %#v, want per-cast warnings followed by two group warnings", result.Warnings)
|
||||||
|
}
|
||||||
|
if result.Warnings[0].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[0].Scope != "spell_casts[0]" {
|
||||||
|
t.Fatalf("warning[0] = %#v, want input name warning", result.Warnings[0])
|
||||||
|
}
|
||||||
|
if result.Warnings[1].ReasonCode != ReasonCodeSourceReferencesNormalized || result.Warnings[1].Scope != "spell_casts[0]" {
|
||||||
|
t.Fatalf("warning[1] = %#v, want input source warning", result.Warnings[1])
|
||||||
|
}
|
||||||
|
if result.Warnings[2].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[2].Scope != "spell_casts[3]" {
|
||||||
|
t.Fatalf("warning[2] = %#v, want removed occurrence warning", result.Warnings[2])
|
||||||
|
}
|
||||||
|
if result.Warnings[3].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[3].Scope != "spell_casts[4]" {
|
||||||
|
t.Fatalf("warning[3] = %#v, want removed occurrence warning", result.Warnings[3])
|
||||||
|
}
|
||||||
|
if result.Warnings[4].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed || result.Warnings[4].Scope != "spell_casts[0]" || !strings.Contains(result.Warnings[4].Message, "retained input index 0") || !strings.Contains(result.Warnings[4].Message, "removed input indices [3]") {
|
||||||
|
t.Fatalf("warning[4] = %#v, want first duplicate group warning", result.Warnings[4])
|
||||||
|
}
|
||||||
|
if result.Warnings[5].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed || result.Warnings[5].Scope != "spell_casts[2]" || !strings.Contains(result.Warnings[5].Message, "removed input indices [4, 5]") {
|
||||||
|
t.Fatalf("warning[5] = %#v, want second duplicate group warning", result.Warnings[5])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeKeepsDistinctAndIneligibleCastsSeparate(t *testing.T) {
|
||||||
|
doc := sourceDocument(6)
|
||||||
|
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
|
||||||
|
otherRef := source.SourceRef{SourceID: "source", StartUnitID: 3, EndUnitID: 4}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
variant dnd.SpellCast
|
||||||
|
}{
|
||||||
|
{name: "distinct canonical spell", variant: dnd.SpellCast{Spell: "Healing Word", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}},
|
||||||
|
{name: "distinct caster", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Borin", SourceRefs: []source.SourceRef{ref}}},
|
||||||
|
{name: "distinct evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{otherRef}}},
|
||||||
|
{name: "unknown spell", variant: dnd.SpellCast{Spell: "Unknown Spell", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}},
|
||||||
|
{name: "empty evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria"}},
|
||||||
|
{name: "invalid evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 2}}}},
|
||||||
|
}
|
||||||
|
base := dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(dnd.SpellList{SpellCasts: []dnd.SpellCast{base, test.variant}}, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(result.Value.SpellCasts) != 2 {
|
||||||
|
t.Fatalf("normalized casts = %#v, want both casts retained", result.Value.SpellCasts)
|
||||||
|
}
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
if warning.ReasonCode == ReasonCodeDuplicateSpellCastCollapsed {
|
||||||
|
t.Fatalf("warnings = %#v, want no duplicate collapse", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeDoesNotCollapseAdjacentOrOverlappingEvidence(t *testing.T) {
|
||||||
|
doc := sourceDocument(8)
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 2}}},
|
||||||
|
{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 3, EndUnitID: 4}}},
|
||||||
|
{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 3}}},
|
||||||
|
{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 2, EndUnitID: 4}}},
|
||||||
|
}}
|
||||||
|
result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(result.Value.SpellCasts) != len(input.SpellCasts) {
|
||||||
|
t.Fatalf("normalized casts = %#v, want adjacent and overlapping evidence retained", result.Value.SpellCasts)
|
||||||
|
}
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
if warning.ReasonCode == ReasonCodeDuplicateSpellCastCollapsed {
|
||||||
|
t.Fatalf("warnings = %#v, want no duplicate collapse", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeBoundsDuplicateWarningIndices(t *testing.T) {
|
||||||
|
doc := sourceDocument(2)
|
||||||
|
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
|
||||||
|
casts := make([]dnd.SpellCast, 22)
|
||||||
|
for index := range casts {
|
||||||
|
casts[index] = dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}
|
||||||
|
}
|
||||||
|
result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(dnd.SpellList{SpellCasts: casts}, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(result.Value.SpellCasts) != 1 || len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed {
|
||||||
|
t.Fatalf("result = %#v, warnings = %#v, want one retained cast and one bounded warning", result.Value, result.Warnings)
|
||||||
|
}
|
||||||
|
message := result.Warnings[0].Message
|
||||||
|
if !strings.Contains(message, "removed input indices [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]") || strings.Contains(message, ", 21]") || !strings.Contains(message, "1 additional removed input indices omitted") {
|
||||||
|
t.Fatalf("warning message = %q, want 20 displayed indices and exact omitted count", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeIsIdempotentForAlreadyNormalizedInput(t *testing.T) {
|
||||||
|
doc := sourceDocument(3)
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||||
|
Spell: "Cure Wounds",
|
||||||
|
Caster: "Aria",
|
||||||
|
Effect: "effect",
|
||||||
|
NarrativeDescription: "narrative",
|
||||||
|
SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 2}},
|
||||||
|
}}}
|
||||||
|
normalizer := newNormalizer(t)
|
||||||
|
first, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
second, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(second.Value, first.Value) || len(first.Warnings) != 0 || len(second.Warnings) != 0 {
|
||||||
|
t.Fatalf("first = %#v/%#v, second = %#v/%#v, want identical artifacts without mutation warnings", first.Value, first.Warnings, second.Value, second.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeDuplicateOutputDoesNotShareInputSlices(t *testing.T) {
|
||||||
|
doc := sourceDocument(3)
|
||||||
|
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
|
||||||
|
input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||||
|
{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}},
|
||||||
|
{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{ref}},
|
||||||
|
}}
|
||||||
|
result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
result.Value.SpellCasts[0].SourceRefs[0].SourceID = "changed"
|
||||||
|
if input.SpellCasts[0].SourceRefs[0].SourceID != "source" || input.SpellCasts[1].SourceRefs[0].SourceID != "source" {
|
||||||
|
t.Fatalf("normalized output shares input references: input = %#v", input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNormalizer(t *testing.T) *Normalizer {
|
||||||
|
t.Helper()
|
||||||
|
normalizer, err := New(Options{}, overlayReference())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return normalizer
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRequest(value dnd.SpellList) contracts.TypedNormalizeRequest[dnd.SpellList] {
|
||||||
|
return contracts.TypedNormalizeRequest[dnd.SpellList]{
|
||||||
|
LaneID: "spells",
|
||||||
|
MergeOutput: contracts.MergeArtifact[dnd.SpellList]{
|
||||||
|
LaneID: "spells",
|
||||||
|
MergerKey: "appendorder",
|
||||||
|
SourceID: "source",
|
||||||
|
Value: value,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRequestWithSource(value dnd.SpellList, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.SpellList] {
|
||||||
|
request := normalizeRequest(value)
|
||||||
|
request.Source = doc
|
||||||
|
return request
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceDocument(unitCount int) *source.SourceDocument {
|
||||||
|
units := make([]source.SourceUnit, unitCount)
|
||||||
|
for index := range units {
|
||||||
|
units[index] = source.SourceUnit{ID: index + 1}
|
||||||
|
}
|
||||||
|
return &source.SourceDocument{ID: "source", Units: units}
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayReference() contracts.ReferenceSet {
|
||||||
|
return spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}]}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func spellCatalogReference(content string) contracts.ReferenceSet {
|
||||||
|
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
spellcatalog.SpellCatalogReferenceSlot: {
|
||||||
|
Items: []contracts.ReferenceItem{{
|
||||||
|
SlotName: spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
MediaType: "application/json",
|
||||||
|
Content: []byte(content),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
}
|
||||||
99
internal/modules/dnd/normalize/spells/testdata/normalizer_accepted_cases.json
vendored
Normal file
99
internal/modules/dnd/normalize/spells/testdata/normalizer_accepted_cases.json
vendored
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
{
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"name": "canonicalizes references and collapses exact duplicate",
|
||||||
|
"input": {
|
||||||
|
"spell_casts": [
|
||||||
|
{
|
||||||
|
"caster": " Aria ",
|
||||||
|
"spell": " cure wounds ",
|
||||||
|
"effect": "first effect",
|
||||||
|
"narrative_description": "first narrative",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 2, "end_unit_id": 2},
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1},
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"caster": "aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "duplicate effect",
|
||||||
|
"narrative_description": "duplicate narrative",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1},
|
||||||
|
{"source_id": "source", "start_unit_id": 2, "end_unit_id": 2}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"caster": "aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "distinct effect",
|
||||||
|
"narrative_description": "distinct narrative",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 2, "end_unit_id": 2}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"spell_casts": [
|
||||||
|
{
|
||||||
|
"caster": " Aria ",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "first effect",
|
||||||
|
"narrative_description": "first narrative",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1},
|
||||||
|
{"source_id": "source", "start_unit_id": 2, "end_unit_id": 2}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"caster": "aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "distinct effect",
|
||||||
|
"narrative_description": "distinct narrative",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 2, "end_unit_id": 2}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"warning_reason_codes": [
|
||||||
|
"spell_name_canonicalized",
|
||||||
|
"source_references_normalized",
|
||||||
|
"duplicate_spell_cast_collapsed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "leaves already normalized output unchanged",
|
||||||
|
"input": {
|
||||||
|
"spell_casts": [
|
||||||
|
{
|
||||||
|
"caster": "Aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "heals an ally",
|
||||||
|
"narrative_description": "Aria restores an ally's wounds.",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"spell_casts": [
|
||||||
|
{
|
||||||
|
"caster": "Aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "heals an ally",
|
||||||
|
"narrative_description": "Aria restores an ally's wounds.",
|
||||||
|
"source_refs": [
|
||||||
|
{"source_id": "source", "start_unit_id": 1, "end_unit_id": 1}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"warning_reason_codes": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||||
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
|
||||||
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
||||||
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
||||||
@@ -38,6 +39,7 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
|||||||
{name: "spell-list appendorder merger", register: func() error {
|
{name: "spell-list appendorder merger", register: func() error {
|
||||||
return appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, appendSpellLists)
|
return appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, appendSpellLists)
|
||||||
}},
|
}},
|
||||||
|
{name: "spells normalizer", register: func() error { return spellnormalize.Register(registries.Normalizers) }},
|
||||||
{name: "spell-list noop normalizer", register: func() error { return noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind) }},
|
{name: "spell-list noop normalizer", register: func() error { return noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind) }},
|
||||||
{name: "spell shape validator", register: func() error { return spellshape.Register(registries.Validators) }},
|
{name: "spell shape validator", register: func() error { return spellshape.Register(registries.Validators) }},
|
||||||
{name: "spell catalog validator", register: func() error { return spellcatalog.Register(registries.Validators) }},
|
{name: "spell catalog validator", register: func() error { return spellcatalog.Register(registries.Validators) }},
|
||||||
@@ -71,6 +73,20 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return fmt.Errorf("register dnd spells validator chain: %w", err)
|
return fmt.Errorf("register dnd spells validator chain: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Module: spellnormalize.Key,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding(validjson.Key),
|
||||||
|
pipeline.Binding(validjsonschema.Key),
|
||||||
|
pipeline.Binding(spellshape.Key),
|
||||||
|
pipeline.Binding(spellcatalog.Key),
|
||||||
|
pipeline.Binding(spellsourcerefs.Key),
|
||||||
|
pipeline.Binding(spellrelatedness.Key),
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("register dnd spells normalize validator chain: %w", err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterAddsDNDFamily(t *testing.T) {
|
func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||||
@@ -22,6 +23,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
}
|
}
|
||||||
assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"})
|
assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"})
|
||||||
assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
||||||
|
assertContainsKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{spellnormalize.Key, pipeline.DefaultNormalizeModule})
|
||||||
assertContainsArtifactKinds(t, registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind})
|
assertContainsArtifactKinds(t, registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
assertContainsKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
assertContainsKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||||
"extract/dnd/spells/catalog",
|
"extract/dnd/spells/catalog",
|
||||||
@@ -42,6 +44,9 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
||||||
}
|
}
|
||||||
|
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
|
t.Fatalf("spell normalize validator chain = %#v, want %#v", got, wantChain)
|
||||||
|
}
|
||||||
assertAssetNamesContain(t, assets.PromptFS, []string{
|
assertAssetNamesContain(t, assets.PromptFS, []string{
|
||||||
"dnd.scenes/dnd.scenes.yaml",
|
"dnd.scenes/dnd.scenes.yaml",
|
||||||
"dnd.scenes/instructions.md",
|
"dnd.scenes/instructions.md",
|
||||||
@@ -66,6 +71,9 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
if spec, ok := registries.Extractors.Spec(spells.Key); !ok || spec.ArtifactKind != dnd.SpellListKind {
|
if spec, ok := registries.Extractors.Spec(spells.Key); !ok || spec.ArtifactKind != dnd.SpellListKind {
|
||||||
t.Fatalf("spell extractor spec = %#v, present = %t; want dnd spell-list artifact", spec, ok)
|
t.Fatalf("spell extractor spec = %#v, present = %t; want dnd spell-list artifact", spec, ok)
|
||||||
}
|
}
|
||||||
|
if spec, ok := registries.Normalizers.Spec(spellnormalize.Key); !ok || spec.ArtifactKind != dnd.SpellListKind || spec.Stage != pipeline.StageNormalize {
|
||||||
|
t.Fatalf("spell normalizer spec = %#v, present = %t; want dnd spell-list artifact", spec, ok)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user