Compare commits
8 Commits
249e49c928
...
fc8e03f98c
| Author | SHA1 | Date | |
|---|---|---|---|
| fc8e03f98c | |||
| 16de4b6437 | |||
| 0f30888b00 | |||
| 3e67be6ac3 | |||
| 5ef027b6f0 | |||
| 666b4bf801 | |||
| d593bfee0a | |||
| b7ad66f0e0 |
33
docs/cli.md
33
docs/cli.md
@@ -45,8 +45,10 @@ Flags:
|
|||||||
Defaults to `./notarius-output`.
|
Defaults to `./notarius-output`.
|
||||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||||
invocation.
|
invocation.
|
||||||
- `--llm-profile id`: override every effective LLM-capable module binding to
|
- `--llm-profile id`: override every effective LLM-capable pipeline module
|
||||||
use one Scriptorium profile ID.
|
binding to use one Scriptorium profile ID. Validator-specific profiles are
|
||||||
|
not overridden. Configured LLM-backed validators with explicit profiles are
|
||||||
|
validated against the configured Scriptorium profile source.
|
||||||
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
||||||
module calls.
|
module calls.
|
||||||
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
||||||
@@ -207,8 +209,31 @@ The production CLI currently registers these module keys:
|
|||||||
- normalize: `noop`
|
- normalize: `noop`
|
||||||
- output: `json`
|
- output: `json`
|
||||||
|
|
||||||
Configured validator module lists are reserved for a future validator-chain
|
## Implemented Production Validators
|
||||||
feature and are rejected by current configuration validation.
|
|
||||||
|
The production CLI currently registers these validator keys:
|
||||||
|
|
||||||
|
- `generic/always_accept`
|
||||||
|
- `generic/always_reject`
|
||||||
|
- `generic/valid_json`
|
||||||
|
- `generic/valid_json_schema`
|
||||||
|
- `extract/dnd/spells/shape`
|
||||||
|
- `extract/dnd/spells/source_refs`
|
||||||
|
- `extract/dnd/spells/source_relatedness`
|
||||||
|
|
||||||
|
The production default chain for the `dnd/spells` extractor is:
|
||||||
|
|
||||||
|
1. `generic/valid_json`
|
||||||
|
2. `generic/valid_json_schema`
|
||||||
|
3. `extract/dnd/spells/shape`
|
||||||
|
4. `extract/dnd/spells/source_refs`
|
||||||
|
5. `extract/dnd/spells/source_relatedness`
|
||||||
|
|
||||||
|
Validator chain overrides are configured on `chunk`, lane `extract`, lane
|
||||||
|
`merge`, and lane `normalize` bindings. Omitted overrides use production
|
||||||
|
defaults, `validators: []` disables validation for that binding, and non-empty
|
||||||
|
lists replace the default chain in configured order. Validator keys are resolved
|
||||||
|
against the registered validator catalog.
|
||||||
|
|
||||||
For YAML structure, Scriptorium profile sources, environment overrides, and
|
For YAML structure, Scriptorium profile sources, environment overrides, and
|
||||||
module binding syntax, see [Configuration](config.md).
|
module binding syntax, see [Configuration](config.md).
|
||||||
|
|||||||
@@ -131,8 +131,9 @@ Artifact lane fields:
|
|||||||
- `extract`: required module binding.
|
- `extract`: required module binding.
|
||||||
- `merge`: optional module binding. Default module is `appendorder`.
|
- `merge`: optional module binding. Default module is `appendorder`.
|
||||||
- `normalize`: optional module binding. Default module is `noop`.
|
- `normalize`: optional module binding. Default module is `noop`.
|
||||||
- `validators`: reserved for future configurable validator chains. Non-empty
|
- `validators`: deprecated lane-level validator list. Non-empty lists are
|
||||||
lists are rejected by current configuration validation.
|
rejected; use `extract.validators`, `merge.validators`, or
|
||||||
|
`normalize.validators`.
|
||||||
- `references`: optional compatibility alias for extractor reference bindings.
|
- `references`: optional compatibility alias for extractor reference bindings.
|
||||||
Lane bindings override pipeline-level bindings for the same slot.
|
Lane bindings override pipeline-level bindings for the same slot.
|
||||||
|
|
||||||
@@ -234,12 +235,32 @@ Binding fields:
|
|||||||
- `options`: optional module-specific settings.
|
- `options`: optional module-specific settings.
|
||||||
- `references`: optional reference bindings. Supported only for `chunk`,
|
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||||
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
||||||
reject this field during validation. Validator bindings are reserved for a
|
reject this field during validation.
|
||||||
future validator-chain feature and are rejected when configured.
|
- `validators`: optional stage-local validator chain override. Supported only
|
||||||
|
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
|
||||||
|
use the production default chain; set `validators: []` to force an empty
|
||||||
|
chain; set a non-empty list to use exactly those validators in configured
|
||||||
|
order.
|
||||||
|
|
||||||
|
Validator bindings use the same shorthand or object module-binding form, but
|
||||||
|
only these fields are supported:
|
||||||
|
|
||||||
|
- `module`: validator key.
|
||||||
|
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
|
||||||
|
- `options`: optional validator-specific settings.
|
||||||
|
|
||||||
|
Validator bindings reject `references`, `retries`, and nested `validators`.
|
||||||
|
During resolution, deterministic validators reject explicit `llm_profile`
|
||||||
|
values.
|
||||||
|
|
||||||
The `--llm-profile` run flag overrides every effective LLM-capable module
|
The `--llm-profile` run flag overrides every effective LLM-capable module
|
||||||
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
|
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
|
||||||
merge, and normalize binding.
|
merge, and normalize binding. It does not override validator-specific
|
||||||
|
`llm_profile` values.
|
||||||
|
|
||||||
|
Configured LLM-backed validators with explicit `llm_profile` values are
|
||||||
|
validated against the configured Scriptorium profile source. Deterministic
|
||||||
|
production validators do not call the LLM and must not set `llm_profile`.
|
||||||
|
|
||||||
## Implemented Production Modules
|
## Implemented Production Modules
|
||||||
|
|
||||||
@@ -253,6 +274,32 @@ merge, and normalize binding.
|
|||||||
| normalize | `noop` | Passes merged raw outputs through unchanged. |
|
| normalize | `noop` | Passes merged raw outputs through unchanged. |
|
||||||
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
||||||
|
|
||||||
|
## Implemented Production Validators
|
||||||
|
|
||||||
|
| Key | Execution | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `generic/always_accept` | deterministic | Accepts returned module output. |
|
||||||
|
| `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. |
|
||||||
|
| `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. |
|
||||||
|
| `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. |
|
||||||
|
| `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-cast JSON payloads. |
|
||||||
|
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
|
||||||
|
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
|
||||||
|
|
||||||
|
The production default chain for the `dnd/spells` extractor is:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
validators:
|
||||||
|
- generic/valid_json
|
||||||
|
- generic/valid_json_schema
|
||||||
|
- extract/dnd/spells/shape
|
||||||
|
- extract/dnd/spells/source_refs
|
||||||
|
- extract/dnd/spells/source_relatedness
|
||||||
|
```
|
||||||
|
|
||||||
|
No other production module currently has a default validator chain. Empty
|
||||||
|
chains approve output by default.
|
||||||
|
|
||||||
The `generic` chunker accepts:
|
The `generic` chunker accepts:
|
||||||
|
|
||||||
- `max_units`: positive integer, default `50`;
|
- `max_units`: positive integer, default `50`;
|
||||||
@@ -309,6 +356,10 @@ Pipeline resolution additionally checks:
|
|||||||
- required module keys are present;
|
- required module keys are present;
|
||||||
- module keys are registered for the expected slot;
|
- module keys are registered for the expected slot;
|
||||||
- module capability requirements are satisfied;
|
- module capability requirements are satisfied;
|
||||||
|
- non-empty validator overrides reference registered validator keys;
|
||||||
|
- deterministic validators do not set `llm_profile`;
|
||||||
|
- LLM-backed validators with explicit `llm_profile` values reference configured
|
||||||
|
Scriptorium profile IDs;
|
||||||
- bound reference slots are declared by selected chunk, extractor, merger, or
|
- bound reference slots are declared by selected chunk, extractor, merger, or
|
||||||
normalizer targets;
|
normalizer targets;
|
||||||
- required reference slots are bound for selected targets.
|
- required reference slots are bound for selected targets.
|
||||||
|
|||||||
@@ -82,6 +82,35 @@ sanitizing the lane ID:
|
|||||||
"normalizer": "noop"
|
"normalizer": "noop"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"validator_chains": [
|
||||||
|
{
|
||||||
|
"stage": "extract",
|
||||||
|
"lane_id": "spells",
|
||||||
|
"module_key": "dnd/spells",
|
||||||
|
"validators": [
|
||||||
|
{
|
||||||
|
"key": "generic/valid_json",
|
||||||
|
"execution_class": "deterministic"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "generic/valid_json_schema",
|
||||||
|
"execution_class": "deterministic"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "extract/dnd/spells/shape",
|
||||||
|
"execution_class": "deterministic"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "extract/dnd/spells/source_refs",
|
||||||
|
"execution_class": "deterministic"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "extract/dnd/spells/source_relatedness",
|
||||||
|
"execution_class": "deterministic"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
"validation_status": "approved",
|
"validation_status": "approved",
|
||||||
"started_at": "2026-01-01T00:00:00Z",
|
"started_at": "2026-01-01T00:00:00Z",
|
||||||
"completed_at": "2026-01-01T00:00:01Z"
|
"completed_at": "2026-01-01T00:00:01Z"
|
||||||
@@ -103,6 +132,12 @@ references.
|
|||||||
`validation_status` is `approved` when no raw outputs were rejected and
|
`validation_status` is `approved` when no raw outputs were rejected and
|
||||||
`rejected` when one or more raw outputs were rejected.
|
`rejected` when one or more raw outputs were rejected.
|
||||||
|
|
||||||
|
`validator_chains` records the resolved validator chain for each validation
|
||||||
|
point. Entries include stage, lane ID when applicable, module key, and validators
|
||||||
|
with key and execution class. Empty chains are recorded with an empty
|
||||||
|
`validators` array, including chains resolved from explicit empty config
|
||||||
|
overrides.
|
||||||
|
|
||||||
`normalized_outputs` summarizes each normalized lane output without embedding
|
`normalized_outputs` summarizes each normalized lane output without embedding
|
||||||
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
||||||
type, and response schema provenance where available.
|
type, and response schema provenance where available.
|
||||||
|
|||||||
@@ -19,9 +19,10 @@ structured output. The response also carries the raw structured output bytes
|
|||||||
returned by the runtime so modules can preserve raw payloads in pipeline stage
|
returned by the runtime so modules can preserve raw payloads in pipeline stage
|
||||||
outputs.
|
outputs.
|
||||||
|
|
||||||
Modules that call the LLM own their prompts, schemas, prompt IDs, validators,
|
Modules that call the LLM own their prompts, schemas, prompt IDs, and
|
||||||
and domain-specific interpretation. Provider adapters should not contain
|
domain-specific interpretation. Validator packages own approve/reject policy,
|
||||||
domain-specific prompt logic.
|
and central catalog mappings decide which validators run by default. Provider
|
||||||
|
adapters should not contain domain-specific prompt logic.
|
||||||
|
|
||||||
Prompt input materials carry source or reference bytes with optional origin
|
Prompt input materials carry source or reference bytes with optional origin
|
||||||
metadata. The Scriptorium-backed runtime receives them as named artifacts rather
|
metadata. The Scriptorium-backed runtime receives them as named artifacts rather
|
||||||
@@ -43,9 +44,11 @@ The runtime records the actual selected Scriptorium profile, provider, and model
|
|||||||
used during execution. Manifest population does not rely on a precomputed
|
used during execution. Manifest population does not rely on a precomputed
|
||||||
profile ID before pipeline execution.
|
profile ID before pipeline execution.
|
||||||
|
|
||||||
Explicit profile validation and `--llm-profile` overrides apply to LLM-capable
|
Explicit profile validation applies to LLM-capable pipeline stages: chunk,
|
||||||
pipeline stages: chunk, extract, merge, and normalize. Input, output, and
|
extract, merge, normalize, and LLM-backed validators with explicit
|
||||||
validator bindings are not part of the current production LLM profile scope.
|
`llm_profile` values. Input, output, and deterministic validators do not call
|
||||||
|
the LLM. The `--llm-profile` run flag overrides effective chunk, extract, merge,
|
||||||
|
and normalize bindings; it does not override validator-specific profiles.
|
||||||
|
|
||||||
## Scriptorium Adapter
|
## Scriptorium Adapter
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,9 @@ contract from `internal/framework/contracts`, exposes a `ModuleSpec`, and
|
|||||||
registers itself with the matching pipeline registry.
|
registers itself with the matching pipeline registry.
|
||||||
|
|
||||||
The CLI production catalog currently registers only the modules listed here.
|
The CLI production catalog currently registers only the modules listed here.
|
||||||
|
Validator implementations live under `internal/validators` and are registered
|
||||||
|
separately from modules. Production default validator chains are central CLI
|
||||||
|
catalog policy; module packages do not own their default validation chains.
|
||||||
|
|
||||||
## Contract Pattern
|
## Contract Pattern
|
||||||
|
|
||||||
@@ -181,6 +184,12 @@ metadata under `artifact_lanes[].metadata.extractor`. Durable raw output
|
|||||||
details belong in the
|
details belong in the
|
||||||
[D&D spell raw output contract](../integrations/dnd-spell-artifacts.md).
|
[D&D spell raw output contract](../integrations/dnd-spell-artifacts.md).
|
||||||
|
|
||||||
|
The production catalog validates `dnd/spells` raw extract output with generic
|
||||||
|
JSON validators followed by D&D spell validators under
|
||||||
|
`internal/validators/extract/dnd/spells`. The extractor itself remains
|
||||||
|
responsible for prompt, schema, and raw output production rather than
|
||||||
|
approve/reject policy.
|
||||||
|
|
||||||
The `dnd/scenes` chunker and `dnd/spells` extractor declare optional `players`,
|
The `dnd/scenes` chunker and `dnd/spells` extractor declare optional `players`,
|
||||||
`party`, and `glossary` reference slots accepting UTF-8 plain text, Markdown,
|
`party`, and `glossary` reference slots accepting UTF-8 plain text, Markdown,
|
||||||
YAML, or JSON. They also accept `roster` as a deprecated compatibility alias for
|
YAML, or JSON. They also accept `roster` as a deprecated compatibility alias for
|
||||||
|
|||||||
@@ -75,7 +75,8 @@ prompt execution metadata.
|
|||||||
|
|
||||||
`pipeline.Registries` holds concrete constructors for execution. A
|
`pipeline.Registries` holds concrete constructors for execution. A
|
||||||
`pipeline.ModuleCatalog` exposes module specs for config validation and
|
`pipeline.ModuleCatalog` exposes module specs for config validation and
|
||||||
resolution.
|
resolution. The catalog also exposes validator specs and central default
|
||||||
|
validator-chain mappings without constructing modules or validators.
|
||||||
|
|
||||||
Every production module registers a `ModuleSpec` with:
|
Every production module registers a `ModuleSpec` with:
|
||||||
|
|
||||||
@@ -91,6 +92,14 @@ instances. Input, validate, and output specs must not declare reference slots.
|
|||||||
|
|
||||||
Capability checks prevent incompatible pipeline composition before a run starts.
|
Capability checks prevent incompatible pipeline composition before a run starts.
|
||||||
|
|
||||||
|
Every production validator registers a `ValidatorSpec` with:
|
||||||
|
|
||||||
|
- `Key`: validator key used in config and manifests;
|
||||||
|
- `ExecutionClass`: `deterministic` or `llm_backed`.
|
||||||
|
|
||||||
|
Default validator chains are keyed by workflow stage and module key. Production
|
||||||
|
currently registers a default chain for `extract` module `dnd/spells` only.
|
||||||
|
|
||||||
## Runner Input And Output
|
## Runner Input And Output
|
||||||
|
|
||||||
`pipeline.RunInput` carries:
|
`pipeline.RunInput` carries:
|
||||||
@@ -118,8 +127,8 @@ The runner:
|
|||||||
2. builds the input adapter and parses the raw input into a source document;
|
2. builds the input adapter and parses the raw input into a source document;
|
||||||
3. validates the source document;
|
3. validates the source document;
|
||||||
4. builds the chunker and produces source chunks, retrying when configured;
|
4. builds the chunker and produces source chunks, retrying when configured;
|
||||||
5. validates source chunks against framework invariants and any registered raw
|
5. validates source chunks against framework invariants and the resolved chunk
|
||||||
chunk validators;
|
validator chain;
|
||||||
6. runs each selected artifact lane in sorted resolved order;
|
6. runs each selected artifact lane in sorted resolved order;
|
||||||
7. builds the output encoder and validates logical output file names.
|
7. builds the output encoder and validates logical output file names.
|
||||||
8. passes accepted normalized raw outputs, rejected output records, warnings,
|
8. passes accepted normalized raw outputs, rejected output records, warnings,
|
||||||
@@ -181,20 +190,38 @@ Within an artifact lane, the runner:
|
|||||||
|
|
||||||
## Validators
|
## Validators
|
||||||
|
|
||||||
The current runner handoff is raw-output based. Extractors, mergers, and
|
The runner handoff is raw-output based. Chunkers, extractors, mergers, and
|
||||||
normalizers do not advertise validator chains through their module interfaces.
|
normalizers do not advertise validator chains through their module interfaces.
|
||||||
Runner-side raw validation chains receive the raw module output plus
|
Resolved validation chains receive the raw module output plus stage, lane,
|
||||||
stage, lane, module, source, and chunk provenance. Empty raw validation chains
|
module, source, chunk, schema, session, reference, LLM client/profile, binding
|
||||||
approve output by default.
|
option, and run metadata context. Chunk validators receive the chunk result
|
||||||
|
collection, merge validators receive the ordered extract outputs used by the
|
||||||
|
merge, and normalize validators receive the accepted merge output. Empty chains
|
||||||
|
approve output by default. Response-schema provenance may include in-memory JSON
|
||||||
|
schema bytes for validators. Those bytes are omitted from manifests,
|
||||||
|
diagnostics, and encoded output files.
|
||||||
|
|
||||||
Pipeline-configured validator lists are not part of the current runner
|
Resolved validator chains come from central default mappings unless a
|
||||||
contract. Non-empty configured validator lists are rejected during configuration
|
stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or
|
||||||
validation or resolved-run validation so they cannot appear in manifests without
|
lane `normalize`. Explicit empty overrides are valid and are recorded as empty
|
||||||
executing.
|
chains in manifests. Explicit non-empty overrides replace the default chain and
|
||||||
|
preserve configured order.
|
||||||
|
|
||||||
|
The production default chain for `extract` module `dnd/spells` is:
|
||||||
|
|
||||||
|
1. `generic/valid_json`
|
||||||
|
2. `generic/valid_json_schema`
|
||||||
|
3. `extract/dnd/spells/shape`
|
||||||
|
4. `extract/dnd/spells/source_refs`
|
||||||
|
5. `extract/dnd/spells/source_relatedness`
|
||||||
|
|
||||||
|
No other production module currently has a default validator chain.
|
||||||
|
|
||||||
Validator rejection is a non-fatal run outcome: the rejected output is recorded
|
Validator rejection is a non-fatal run outcome: the rejected output is recorded
|
||||||
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution
|
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution
|
||||||
errors are framework-level errors and retry according to the relevant binding.
|
errors are framework-level errors and retry according to the relevant binding.
|
||||||
|
Warning-only validators return approved results with warnings; those warnings
|
||||||
|
are promoted only from successful attempts whose outputs are used.
|
||||||
|
|
||||||
## Warnings And Failures
|
## Warnings And Failures
|
||||||
|
|
||||||
|
|||||||
@@ -110,9 +110,8 @@ stderr, and writes warnings to durable output and diagnostics when retained.
|
|||||||
The run manifest `validation_status` indicates whether raw outputs were
|
The run manifest `validation_status` indicates whether raw outputs were
|
||||||
approved or rejected after validation.
|
approved or rejected after validation.
|
||||||
|
|
||||||
Reference-related warnings include empty bound reference files and D&D spell
|
Reference-related warnings include empty bound reference files. Empty references
|
||||||
relatedness warnings such as `spell_not_near_source`. Empty references are still
|
are still passed to extractors so optional slots can be intentionally blank.
|
||||||
passed to extractors so optional slots can be intentionally blank.
|
|
||||||
|
|
||||||
## Cleanup
|
## Cleanup
|
||||||
|
|
||||||
|
|||||||
@@ -82,8 +82,9 @@ format-specific validation rules. They should not own extraction-domain
|
|||||||
decisions.
|
decisions.
|
||||||
|
|
||||||
Extract modules own artifact semantics, prompt usage, structured response schema
|
Extract modules own artifact semantics, prompt usage, structured response schema
|
||||||
selection, validator defaults, and domain-specific interpretation. They should
|
selection, and domain-specific interpretation. They should depend on framework
|
||||||
depend on framework contracts and core source/artifact types, not concrete input
|
contracts and core source/artifact types, not concrete input module packages.
|
||||||
|
Production validation defaults are central catalog policy, not behavior owned by
|
||||||
module packages.
|
module packages.
|
||||||
|
|
||||||
Merge modules combine extracted candidates. Normalize modules reconcile merged
|
Merge modules combine extracted candidates. Normalize modules reconcile merged
|
||||||
@@ -99,14 +100,19 @@ warnings.
|
|||||||
|
|
||||||
Validators should be independently testable and composable.
|
Validators should be independently testable and composable.
|
||||||
|
|
||||||
Deterministic validators should run before LLM-backed validators when both are
|
Validators evaluate immutable module outputs returned by `chunk`, `extract`,
|
||||||
present. Validator decision semantics should be explicit: each candidate
|
`merge`, and `normalize` stages. Validator decision semantics should be
|
||||||
artifact evaluated by a validator should receive exactly one decision from that
|
explicit: each validator call approves, rejects, or approves with warnings for
|
||||||
validator.
|
the whole module output it receives. Validator rejection records rejected raw
|
||||||
|
output; validator execution errors are framework errors.
|
||||||
|
|
||||||
LLM-backed review belongs in module-owned validator chains, not in an implicit
|
Default validator chains belong in central production catalog mappings keyed by
|
||||||
global review phase. Extract and normalize modules may both use deterministic
|
stage and module key. Pipeline configuration may override those mappings at the
|
||||||
and LLM-backed validators.
|
stage-local module binding. Empty chains are valid and approve by default.
|
||||||
|
|
||||||
|
Deterministic validators should run before LLM-backed validators in production
|
||||||
|
defaults when both are present. Configured validator order is authoritative and
|
||||||
|
must not be silently reordered.
|
||||||
|
|
||||||
Shared validator runtime mechanics belong in framework code. Concrete validator
|
Shared validator runtime mechanics belong in framework code. Concrete validator
|
||||||
behavior belongs in module or validator implementation packages.
|
behavior belongs in module or validator implementation packages.
|
||||||
|
|||||||
@@ -20,7 +20,8 @@ future work only.
|
|||||||
if references become large enough to require preprocessing.
|
if references become large enough to require preprocessing.
|
||||||
- Cross-lane entity normalization.
|
- Cross-lane entity normalization.
|
||||||
- Cross-chunk semantic deduplication.
|
- Cross-chunk semantic deduplication.
|
||||||
- Configurable validator chains with production validator modules.
|
- Additional validator packages and production default chains for future
|
||||||
|
modules.
|
||||||
- Parallel execution where it preserves deterministic manifests and diagnostics.
|
- Parallel execution where it preserves deterministic manifests and diagnostics.
|
||||||
- Additional output encoders.
|
- Additional output encoders.
|
||||||
|
|
||||||
|
|||||||
@@ -1,486 +1,16 @@
|
|||||||
# Validation Refactor Implementation Plan
|
# Validation Refactor Implementation
|
||||||
|
|
||||||
This plan implements the target state described in
|
The validation refactor described by this temporary implementation plan has
|
||||||
[validation.md](validation.md). Follow the stages in order. Do not skip the
|
been completed.
|
||||||
focused tests for a stage before moving to the next one.
|
|
||||||
|
|
||||||
## Stage 1: Replace Legacy Validator Contracts With One Raw Output Contract
|
Current behavior is documented in:
|
||||||
|
|
||||||
Goal: make the raw module-output validation boundary the only framework
|
- [Configuration](../config.md)
|
||||||
validator contract.
|
- [CLI Reference](../cli.md)
|
||||||
|
- [Pipeline Internals](../internal/pipeline.md)
|
||||||
|
- [Modules](../internal/modules.md)
|
||||||
|
- [JSON Output](../integrations/json-output.md)
|
||||||
|
- [Troubleshooting](../troubleshooting.md)
|
||||||
|
|
||||||
Tasks:
|
Remaining future validation ideas, if any, belong in
|
||||||
|
[Validation Roadmap](validation.md).
|
||||||
- In `internal/framework/contracts`, replace the legacy candidate-oriented
|
|
||||||
validator contract with a module-output contract.
|
|
||||||
- Remove these legacy types after updating callers:
|
|
||||||
- `ValidationDecision`;
|
|
||||||
- candidate-oriented `ValidationRequest`;
|
|
||||||
- candidate-oriented `ValidationResult`;
|
|
||||||
- legacy `Validator` behavior that works over `artifacts.ArtifactCandidate`.
|
|
||||||
- Remove `RawValidator`, `RawValidationRequest`, and `RawValidationResult`.
|
|
||||||
The new `Validator` contract covers the current raw validation use case
|
|
||||||
directly.
|
|
||||||
- Define execution class metadata:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ExecutionClass string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ExecutionClassDeterministic ExecutionClass = "deterministic"
|
|
||||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
- Define the new request shape around immutable module output. It should carry:
|
|
||||||
- `Stage` as a string-compatible stage identifier;
|
|
||||||
- `LaneID`;
|
|
||||||
- `ModuleKey`;
|
|
||||||
- `Source` and `SourceID`;
|
|
||||||
- `SourceInput`;
|
|
||||||
- `SessionID`;
|
|
||||||
- `References`;
|
|
||||||
- `LLMClient`;
|
|
||||||
- `LLMProfile`;
|
|
||||||
- `Options`;
|
|
||||||
- `Metadata`;
|
|
||||||
- `Schema`;
|
|
||||||
- `Payload`;
|
|
||||||
- chunk provenance for extract validation;
|
|
||||||
- `Chunks` for chunk validation;
|
|
||||||
- ordered `ExtractOutputs` for merge validation;
|
|
||||||
- `MergeOutput` for normalize validation.
|
|
||||||
- Define the result shape as one decision over the whole module output:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ValidationResult struct {
|
|
||||||
Approved bool
|
|
||||||
ReasonCode string
|
|
||||||
Message string
|
|
||||||
DiagnosticArtifactPath string
|
|
||||||
Warnings []Warning
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- Define `Validator` as:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Validator interface {
|
|
||||||
Name() string
|
|
||||||
ExecutionClass() ExecutionClass
|
|
||||||
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- Update tests under `internal/framework/contracts` so fake validators exercise
|
|
||||||
the new contract and no longer reference artifact candidates.
|
|
||||||
- Keep `artifacts.ArtifactCandidate` only if other non-validation code still
|
|
||||||
needs it. If it becomes unused after later stages, remove it in Stage 6.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- Validators must not mutate request payloads, source documents, chunks, or
|
|
||||||
upstream outputs.
|
|
||||||
- Validator execution errors are framework errors. Validator semantic rejection
|
|
||||||
is represented by `ValidationResult{Approved:false}`.
|
|
||||||
- Empty validation chains still approve output by default.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/framework/contracts
|
|
||||||
go test ./internal/framework/validate
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 2: Add Validator Specs, Chain Mappings, And Manifest Provenance
|
|
||||||
|
|
||||||
Goal: make validators centrally registered and make default chains explicit,
|
|
||||||
reviewable, and auditable.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Replace the current `pipeline.ValidatorRegistry` internals so it registers
|
|
||||||
validator specs, not generic module specs.
|
|
||||||
- Add a `pipeline.ValidatorSpec` with at least:
|
|
||||||
- `Key`;
|
|
||||||
- `ExecutionClass`.
|
|
||||||
- Keep validator construction lazy through `ValidatorRegistry.Build(key)`.
|
|
||||||
- Add `ValidatorRegistry.Spec(key)` and sorted registered-spec accessors for
|
|
||||||
catalog and manifest use.
|
|
||||||
- Remove `pipeline.RawValidationRegistry` after replacing its callers with the
|
|
||||||
new chain registry. The new chain registry should be keyed by:
|
|
||||||
- stage;
|
|
||||||
- module key.
|
|
||||||
- Add a production/default mapping type such as:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ValidatorChainMapping struct {
|
|
||||||
Stage ModuleStage
|
|
||||||
Module string
|
|
||||||
Validators []ModuleBinding
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- Add a chain registry with these behaviors:
|
|
||||||
- duplicate stage/module mappings are rejected;
|
|
||||||
- unknown stages are rejected;
|
|
||||||
- empty default chains may be represented by absence of a mapping;
|
|
||||||
- lookup returns a defensive copy.
|
|
||||||
- Extend `pipeline.ModuleCatalog` and `pipeline.Registries` to carry the chain
|
|
||||||
mapping registry alongside module and validator registries.
|
|
||||||
- Add manifest types under `internal/core/artifacts` for resolved validator
|
|
||||||
chain provenance. Use a top-level manifest field so chunk, extract, merge, and
|
|
||||||
normalize chains can all be represented:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ValidatorChainManifest struct {
|
|
||||||
Stage string `json:"stage"`
|
|
||||||
LaneID string `json:"lane_id,omitempty"`
|
|
||||||
ModuleKey string `json:"module_key"`
|
|
||||||
Validators []ValidatorManifest `json:"validators"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ValidatorManifest struct {
|
|
||||||
Key string `json:"key"`
|
|
||||||
ExecutionClass string `json:"execution_class"`
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- Remove `ArtifactLaneManifest.Validators` and update affected tests and JSON
|
|
||||||
output documentation to use top-level validator-chain provenance instead.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- The production chain mapping is central catalog policy, not module-owned
|
|
||||||
behavior.
|
|
||||||
- A resolved run manifest must record the exact chain used for each validation
|
|
||||||
point, including explicit empty chains.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
go test ./internal/core/artifacts
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 3: Implement Stage-Scoped Config Overrides
|
|
||||||
|
|
||||||
Goal: allow pipeline config to override default validator mappings for each
|
|
||||||
validatable stage while preserving unset versus explicit empty semantics.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Add stage-local validator override syntax to module bindings for only:
|
|
||||||
- `chunk`;
|
|
||||||
- artifact lane `extract`;
|
|
||||||
- artifact lane `merge`;
|
|
||||||
- artifact lane `normalize`.
|
|
||||||
- Do not revive artifact-lane-level `validators` as an active chain. Keep that
|
|
||||||
field rejected with an error that directs users to
|
|
||||||
`extract.validators`, `merge.validators`, or `normalize.validators`.
|
|
||||||
- Update file-config parsing so the implementation can distinguish:
|
|
||||||
- validators omitted;
|
|
||||||
- `validators: []`;
|
|
||||||
- `validators: [ ... ]`.
|
|
||||||
- Represent this in pipeline profiles with this explicit override type:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ValidatorOverride struct {
|
|
||||||
Set bool
|
|
||||||
Validators []ModuleBinding
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- Validate configured validators during config validation:
|
|
||||||
- validator module key must be non-empty;
|
|
||||||
- `references` are not supported on validator bindings;
|
|
||||||
- nested validator overrides inside validator bindings are not supported;
|
|
||||||
- retries are not supported on validator bindings in this pass;
|
|
||||||
- explicit `llm_profile` is allowed only for LLM-backed validators and is
|
|
||||||
rejected for deterministic validators during resolution.
|
|
||||||
- During pipeline resolution:
|
|
||||||
- unset override uses the central default chain;
|
|
||||||
- explicit empty override resolves to an empty chain and approves by default;
|
|
||||||
- explicit non-empty override resolves exactly the configured validators in
|
|
||||||
configured order;
|
|
||||||
- every referenced validator key must exist in the validator registry;
|
|
||||||
- configured order must not be silently reordered.
|
|
||||||
- Update `--llm-profile` behavior:
|
|
||||||
- continue overriding chunk, extract, merge, and normalize module bindings;
|
|
||||||
- do not override validator bindings unless validator-specific profile
|
|
||||||
override behavior is explicitly added in a later roadmap.
|
|
||||||
- Update Scriptorium explicit profile validation so it includes LLM-backed
|
|
||||||
validators with explicit `llm_profile` values, and ignores deterministic
|
|
||||||
validators.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- Validator compatibility with a module or stage is not enforced during config
|
|
||||||
resolution.
|
|
||||||
- Empty resolved chains are valid and should be recorded in manifest
|
|
||||||
provenance.
|
|
||||||
- Config validation must fail before runtime if a non-empty override references
|
|
||||||
an unknown validator.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/core/config
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
go test ./internal/cli
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 4: Wire Validator Execution Into The Runner
|
|
||||||
|
|
||||||
Goal: run resolved validator chains at chunk, extract, merge, and normalize
|
|
||||||
validation points.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Replace runner calls to the old raw validation registry with calls to the
|
|
||||||
resolved validator chain for the current validation point.
|
|
||||||
- Build validators from `ValidatorRegistry` at runtime in resolved order.
|
|
||||||
- For each validator call, populate the new `contracts.ValidationRequest` with:
|
|
||||||
- immutable module output payload;
|
|
||||||
- stage/module/lane/source/chunk provenance;
|
|
||||||
- schema metadata and in-memory schema content when available;
|
|
||||||
- source input material;
|
|
||||||
- session ID;
|
|
||||||
- resolved references for the target stage;
|
|
||||||
- LLM client/profile/options/metadata for the validator;
|
|
||||||
- upstream outputs needed by merge and normalize validators.
|
|
||||||
- Preserve current retry behavior:
|
|
||||||
- semantic rejection returns a `RejectedOutput` and may be retried according
|
|
||||||
to the module binding's retry setting;
|
|
||||||
- validator execution errors are framework errors and are retried under the
|
|
||||||
same module binding retry setting;
|
|
||||||
- final rejection is recorded and does not pass downstream.
|
|
||||||
- Preserve current warning behavior:
|
|
||||||
- warnings from accepted attempts are promoted;
|
|
||||||
- warnings from discarded retry attempts are not promoted;
|
|
||||||
- warning-only validators return `Approved:true` with warnings.
|
|
||||||
- Continue enforcing framework-level chunk invariants in the runner before
|
|
||||||
chunk validators run.
|
|
||||||
- Populate manifest validator-chain provenance after resolution and before
|
|
||||||
output encoding.
|
|
||||||
- Keep rejected output manifest entries compatible with current `rejected.json`
|
|
||||||
and `manifest.json` shapes where practical.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- Validators are read-only. Do not let a validator return rewritten payloads or
|
|
||||||
replacement stage outputs.
|
|
||||||
- If a validator chain is empty, approve without building validators.
|
|
||||||
- If a validator is mapped to an unsuitable output shape, the validator should
|
|
||||||
return a clear execution error or an explicit documented approval.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
go test ./internal/cli
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 5: Add Generic Validators
|
|
||||||
|
|
||||||
Goal: provide reusable validators needed for incremental module development and
|
|
||||||
production structural checks.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Add `internal/validators/generic/always_accept`.
|
|
||||||
- Key: `generic/always_accept`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Always returns approved.
|
|
||||||
- Add `internal/validators/generic/always_reject`.
|
|
||||||
- Key: `generic/always_reject`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Always returns rejected with stable reason code `always_reject`.
|
|
||||||
- Add `internal/validators/generic/valid_json`.
|
|
||||||
- Key: `generic/valid_json`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Accepts syntactically valid JSON.
|
|
||||||
- Rejects invalid JSON with stable reason code `invalid_json`.
|
|
||||||
- Add `internal/validators/generic/valid_json_schema`.
|
|
||||||
- Key: `generic/valid_json_schema`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Uses the module output's in-memory JSON schema content.
|
|
||||||
- Rejects invalid JSON with `invalid_json`.
|
|
||||||
- Rejects schema non-conformance with `json_schema_invalid`.
|
|
||||||
- Returns a validator execution error when no schema content is available.
|
|
||||||
- Make `github.com/santhosh-tekuri/jsonschema/v6` a direct dependency for
|
|
||||||
schema validation.
|
|
||||||
- Extend `contracts.ResponseSchema` with an in-memory-only field:
|
|
||||||
|
|
||||||
```go
|
|
||||||
JSONSchema []byte `json:"-"`
|
|
||||||
```
|
|
||||||
|
|
||||||
Module outputs should carry schema bytes to validators without serializing raw
|
|
||||||
schema content into manifests, diagnostics, or default output files.
|
|
||||||
- Update D&D scene and D&D spell modules to populate in-memory schema content on
|
|
||||||
their output schema values.
|
|
||||||
- Do not add a shared validator test helper package in this pass. Keep tests
|
|
||||||
local unless a later cleanup finds substantial duplication.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- Generic validators must not depend on concrete production modules.
|
|
||||||
- Tests must not assert exact embedded production prompt text.
|
|
||||||
- Raw schema content must not be emitted in diagnostics or manifests.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/validators/generic/...
|
|
||||||
go test ./internal/modules/chunk/dnd/scenes
|
|
||||||
go test ./internal/modules/extract/dnd/spells
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 6: Migrate D&D Spell Validators
|
|
||||||
|
|
||||||
Goal: move D&D spell validation behavior out of the extractor package and into
|
|
||||||
raw-output validators.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Add `internal/validators/extract/dnd/spells/shape`.
|
|
||||||
- Key: `extract/dnd/spells/shape`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Parses raw spell JSON and rejects malformed spell-cast payloads or missing
|
|
||||||
required fields.
|
|
||||||
- Add `internal/validators/extract/dnd/spells/source_refs`.
|
|
||||||
- Key: `extract/dnd/spells/source_refs`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Parses raw spell JSON and rejects missing or invalid source references.
|
|
||||||
- Use `internal/core/source.ValidateRef` for source reference validation.
|
|
||||||
- Add `internal/validators/extract/dnd/spells/source_relatedness`.
|
|
||||||
- Key: `extract/dnd/spells/source_relatedness`.
|
|
||||||
- Execution class: deterministic.
|
|
||||||
- Warning-only validator.
|
|
||||||
- Approves output and emits `spell_not_near_source` warnings when a spell name
|
|
||||||
is not found in the cited source text.
|
|
||||||
- Keep D&D-specific validator parsing local to validator packages or a validator
|
|
||||||
helper package. Do not import concrete extractor packages from validators.
|
|
||||||
- Remove `internal/modules/extract/dnd/spells/validator.go` and its
|
|
||||||
candidate-oriented tests after equivalent validator-package tests exist.
|
|
||||||
- Remove now-unused legacy candidate validation helpers and artifact candidate
|
|
||||||
types if they have no remaining callers.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- The spell extractor should remain responsible for prompt/schema/provenance
|
|
||||||
and raw output production, not accept/reject policy.
|
|
||||||
- `source_relatedness` must not reject output. It should only warn.
|
|
||||||
- The default D&D spell chain must preserve the intended order:
|
|
||||||
`generic/valid_json`, `generic/valid_json_schema`,
|
|
||||||
`extract/dnd/spells/shape`, `extract/dnd/spells/source_refs`,
|
|
||||||
`extract/dnd/spells/source_relatedness`.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/validators/extract/dnd/spells/...
|
|
||||||
go test ./internal/modules/extract/dnd/spells
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 7: Register Production Validators And Defaults
|
|
||||||
|
|
||||||
Goal: make production validation policy explicit and active.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Register all generic validators in `internal/cli/catalog.go`.
|
|
||||||
- Register all D&D spell validators in `internal/cli/catalog.go`.
|
|
||||||
- Register default production chain mappings centrally near production module
|
|
||||||
registration.
|
|
||||||
- Add the default chain for `extract` module `dnd/spells`:
|
|
||||||
- `generic/valid_json`;
|
|
||||||
- `generic/valid_json_schema`;
|
|
||||||
- `extract/dnd/spells/shape`;
|
|
||||||
- `extract/dnd/spells/source_refs`;
|
|
||||||
- `extract/dnd/spells/source_relatedness`.
|
|
||||||
- Do not add default chains for other modules unless the stage has a meaningful
|
|
||||||
validator for that output shape.
|
|
||||||
- Update catalog construction tests so available modules, available validators,
|
|
||||||
and default mappings can be reviewed together.
|
|
||||||
- Add CLI tests proving:
|
|
||||||
- default D&D spell validators run;
|
|
||||||
- explicit empty override disables the default chain;
|
|
||||||
- explicit non-empty override replaces the default chain;
|
|
||||||
- configured validator order is preserved;
|
|
||||||
- unknown configured validator keys fail during validation/resolution;
|
|
||||||
- deterministic validators reject explicit `llm_profile`;
|
|
||||||
- LLM-backed validator profile validation checks explicit profile IDs.
|
|
||||||
|
|
||||||
Design requirements:
|
|
||||||
|
|
||||||
- Production defaults must be reviewable without constructing concrete modules.
|
|
||||||
- Empty default chains approve by default.
|
|
||||||
- The manifest records resolved chains for default chains, empty overrides, and
|
|
||||||
explicit override chains.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/cli
|
|
||||||
go test ./internal/core/config
|
|
||||||
go test ./internal/framework/pipeline
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 8: Documentation, Examples, And Cleanup
|
|
||||||
|
|
||||||
Goal: make implemented validation behavior canonical outside roadmap docs and
|
|
||||||
remove stale roadmap instructions.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Update `docs/policy/architecture.md` so validation policy reflects central
|
|
||||||
mappings and validator packages rather than module-owned validator chains.
|
|
||||||
- Update `docs/config.md` to document stage-local validator overrides:
|
|
||||||
- unset uses defaults;
|
|
||||||
- explicit empty disables validators;
|
|
||||||
- explicit non-empty replaces defaults in configured order.
|
|
||||||
- Update `docs/cli.md` to list production validators and describe validation
|
|
||||||
profile behavior.
|
|
||||||
- Update `docs/internal/pipeline.md` with the implemented validator contract,
|
|
||||||
chain mapping, retry behavior, and manifest provenance.
|
|
||||||
- Update `docs/internal/modules.md` to remove stale claims that modules own
|
|
||||||
validator defaults.
|
|
||||||
- Update `docs/integrations/json-output.md` for any manifest shape changes.
|
|
||||||
- Update `docs/troubleshooting.md` with validator-chain debugging guidance.
|
|
||||||
- Leave maintained example configs unchanged unless a validator override example
|
|
||||||
is added intentionally with corresponding CLI/config tests. Keep examples
|
|
||||||
secret-free and loadable.
|
|
||||||
- Replace `docs/roadmap/implementation.md` with a concise completed note after
|
|
||||||
all stages are implemented.
|
|
||||||
- Update `docs/roadmap/validation.md` so it no longer describes completed work
|
|
||||||
as future work. Keep only deferred validation work, if any remains.
|
|
||||||
|
|
||||||
Focused documentation checks:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rg -n "reserved for future configurable validator chains|configured validators are not supported|candidate-oriented|RawValidationRegistry" docs internal
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected result after implementation: no stale current-behavior documentation
|
|
||||||
describes configured validators as rejected, and no production code path depends
|
|
||||||
on the legacy candidate-oriented validator contract.
|
|
||||||
|
|
||||||
## Full Validation
|
|
||||||
|
|
||||||
Run after every stage that changes shared contracts, and at the end:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./cmd/notarius
|
|
||||||
```
|
|
||||||
|
|
||||||
Also run focused packages added by this plan:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/validators/...
|
|
||||||
```
|
|
||||||
|
|||||||
@@ -1,463 +1,31 @@
|
|||||||
# Validation System Refactor
|
# Validation Roadmap
|
||||||
|
|
||||||
This roadmap defines the target state for making validation a first-class,
|
The first-class raw-output validation system is implemented. Current behavior is
|
||||||
composable pipeline concern.
|
documented outside roadmap files, especially in [Configuration](../config.md),
|
||||||
|
[CLI Reference](../cli.md), [Pipeline Internals](../internal/pipeline.md), and
|
||||||
Current pipeline behavior is raw-output based. The runner can execute
|
[JSON Output](../integrations/json-output.md).
|
||||||
`contracts.RawValidator` chains from `pipeline.RawValidationRegistry` for
|
|
||||||
`chunk`, `extract`, `merge`, and `normalize` outputs. Empty chains approve by
|
Implemented behavior includes:
|
||||||
default, validator rejection records a rejected raw output, and rejected output
|
|
||||||
does not pass to the next stage. Production currently registers no raw
|
- a single raw module-output validator contract for chunk, extract, merge, and
|
||||||
validators, and non-empty pipeline-configured validator lists are rejected so
|
normalize outputs;
|
||||||
they cannot appear in manifests without executing.
|
- validator specs with deterministic and LLM-backed execution classes;
|
||||||
|
- central production validator registration and default chain mappings;
|
||||||
Legacy candidate validator contracts and D&D spell validators still exist under
|
- stage-local config overrides that distinguish omitted, explicit empty, and
|
||||||
`internal/modules/extract/dnd/spells`, but they are not part of the current
|
explicit non-empty validator chains;
|
||||||
runner path. The desired end state is that validator implementations, validator
|
- manifest provenance for resolved validator chains;
|
||||||
registration, and default module-to-validator mappings are explicit, reviewable,
|
- generic JSON validators and D&D spell validators under `internal/validators`;
|
||||||
and independent of concrete module packages.
|
- production defaults for the `dnd/spells` extractor.
|
||||||
|
|
||||||
## Goals
|
## Future Work
|
||||||
|
|
||||||
- Move artifact and module-output validation behavior out of `internal/modules`
|
- Add production LLM-backed validators when there is a concrete review policy
|
||||||
and into `internal/validators`.
|
that benefits from model judgment.
|
||||||
- Retire or replace the legacy candidate-oriented `contracts.Validator`,
|
- Add validator diagnostics and timing summaries if operators need more detail
|
||||||
`ValidationRequest`, and `ValidationResult` path after equivalent raw-output
|
than `manifest.json`, `rejected.json`, and `warnings.json` provide.
|
||||||
validators exist.
|
- Add media-type validators for non-JSON module outputs when such modules are
|
||||||
- Keep each validator in its own package.
|
introduced.
|
||||||
- Mirror the stage and domain shape of `internal/modules` where a validator is
|
- Add compatibility metadata only if real deployments need config-time
|
||||||
module-specific.
|
enforcement that a validator is suitable for a specific stage or module.
|
||||||
- Support deterministic and LLM-backed validators through the same framework
|
- Add batching or context-window controls for LLM-backed validators if validator
|
||||||
contract.
|
inputs become large enough to require them.
|
||||||
- Allow validators to be mapped to modules at any pipeline stage that returns
|
|
||||||
module output for validation: `chunk`, `extract`, `merge`, or `normalize`.
|
|
||||||
- Make default production module-to-validator mappings centralized and
|
|
||||||
human-readable.
|
|
||||||
- Allow pipeline configuration to override default mappings for advanced use.
|
|
||||||
- Preserve the distinction between an unset validator override and an explicit
|
|
||||||
empty validator override.
|
|
||||||
- Treat an empty validator set as valid and equivalent to approval.
|
|
||||||
- Preserve the rule that module output passes forward unless a validator rejects
|
|
||||||
it.
|
|
||||||
- Make successfully returned module output the explicit validation boundary:
|
|
||||||
questions about output syntax, media type, schema conformance, and domain
|
|
||||||
acceptability should be answered by validators.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
- Do not create a general workflow engine or arbitrary validation DAG.
|
|
||||||
- Do not revive the legacy artifact-candidate validation model as the primary
|
|
||||||
runner path.
|
|
||||||
- Do not enforce validator compatibility with a module or stage in this pass.
|
|
||||||
- Do not move ordinary runtime invariant checks into validator packages.
|
|
||||||
- Do not require every module to have validators.
|
|
||||||
- Do not require LLM-backed validators for modules that can be checked
|
|
||||||
deterministically.
|
|
||||||
- Do not silently reorder configured validator chains unless that behavior is
|
|
||||||
introduced deliberately and documented as part of the validator contract.
|
|
||||||
|
|
||||||
## Validation Boundary
|
|
||||||
|
|
||||||
Validation packages should own approve/reject/warning evaluation of successfully
|
|
||||||
returned module outputs. This means logic that decides whether a chunk result or
|
|
||||||
raw extract, merge, or normalize payload should continue through the pipeline
|
|
||||||
belongs in `internal/validators`.
|
|
||||||
|
|
||||||
The boundary is:
|
|
||||||
|
|
||||||
- no module output was returned: execution failed, and the pipeline should report
|
|
||||||
a module or runtime error;
|
|
||||||
- module output was returned: the validator chain decides whether that output is
|
|
||||||
acceptable, and an empty validator chain approves it.
|
|
||||||
|
|
||||||
Scriptorium and provider errors are execution failures rather than validator
|
|
||||||
rejections. This includes provider timeouts, authentication failures,
|
|
||||||
transport/runtime failures, Scriptorium structured-output retry exhaustion, and
|
|
||||||
malformed responses that Scriptorium rejects before returning module output.
|
|
||||||
|
|
||||||
Other validation-like checks should remain with their owning packages:
|
|
||||||
|
|
||||||
- input parsing and source-format validation stay in input modules;
|
|
||||||
- source document and source reference invariants stay in `internal/core/source`;
|
|
||||||
- generic framework chunk invariants that make extraction possible stay in the
|
|
||||||
runner, such as non-empty chunk content, valid unit ranges, and canonical
|
|
||||||
source-unit ordering;
|
|
||||||
- config validation stays in `internal/core/config`;
|
|
||||||
- registry, profile, and pipeline consistency checks stay in framework and CLI
|
|
||||||
code;
|
|
||||||
- response schema loading stays in module asset code;
|
|
||||||
- Scriptorium runtime errors stay in LLM runtime code.
|
|
||||||
|
|
||||||
Domain validators may call reusable core helpers such as `source.ValidateRef`,
|
|
||||||
but the module-output approval or rejection decision should be made by a
|
|
||||||
validator.
|
|
||||||
|
|
||||||
Validators should answer module-output questions such as:
|
|
||||||
|
|
||||||
- is returned content syntactically valid JSON;
|
|
||||||
- does returned JSON conform to the module's declared schema;
|
|
||||||
- does the returned media type match the module or pipeline policy;
|
|
||||||
- are required domain fields present and non-empty;
|
|
||||||
- are source references valid and appropriately grounded;
|
|
||||||
- does domain-specific output satisfy the configured policy.
|
|
||||||
|
|
||||||
## Audita Patterns To Adapt
|
|
||||||
|
|
||||||
The validator architecture should adapt useful patterns from
|
|
||||||
[`audita`](https://gitea.maximumdirect.net/eric/audita) without copying its
|
|
||||||
narrower transcript-correction shape directly.
|
|
||||||
|
|
||||||
Useful patterns:
|
|
||||||
|
|
||||||
- concrete validators live under `internal/validators`;
|
|
||||||
- shared validator runtime mechanics live under a framework package;
|
|
||||||
- built-in validator keys are stable and centrally registered;
|
|
||||||
- built-in chains are centrally reviewable;
|
|
||||||
- validators carry execution-class metadata;
|
|
||||||
- deterministic and LLM-backed validators implement one contract;
|
|
||||||
- LLM-backed validator runtime can share batching, diagnostics, structured
|
|
||||||
response handling, and malformed-response policy;
|
|
||||||
- reports and manifests can classify validator decisions by execution class.
|
|
||||||
|
|
||||||
Important Notarius differences:
|
|
||||||
|
|
||||||
- mappings must be keyed by stage and module key, not module key alone;
|
|
||||||
- mappings should be owned by the central production catalog, not resolved inside
|
|
||||||
concrete module constructors;
|
|
||||||
- configured mapping order should be authoritative unless the config explicitly
|
|
||||||
opts into a different ordering policy;
|
|
||||||
- validators must support chunk, extract, merge, and normalize outputs rather
|
|
||||||
than only one proposal shape.
|
|
||||||
|
|
||||||
## Validator Package Layout
|
|
||||||
|
|
||||||
Concrete validators should live under `internal/validators`. Module-specific
|
|
||||||
validators should mirror the module tree and use one package per validator:
|
|
||||||
|
|
||||||
```text
|
|
||||||
internal/validators/extract/dnd/spells/shape
|
|
||||||
internal/validators/extract/dnd/spells/source_refs
|
|
||||||
internal/validators/extract/dnd/spells/source_relatedness
|
|
||||||
```
|
|
||||||
|
|
||||||
D&D spell validation policy belongs under
|
|
||||||
`internal/validators/extract/dnd/spells`, split by concern rather than bundled
|
|
||||||
inside the extractor module.
|
|
||||||
|
|
||||||
Generic validators may live under stage-specific generic paths when they operate
|
|
||||||
on a particular stage output shape:
|
|
||||||
|
|
||||||
```text
|
|
||||||
internal/validators/chunk/generic/...
|
|
||||||
internal/validators/extract/generic/...
|
|
||||||
internal/validators/merge/generic/...
|
|
||||||
internal/validators/normalize/generic/...
|
|
||||||
```
|
|
||||||
|
|
||||||
Truly stage-independent validators may live under `internal/validators/generic`
|
|
||||||
once there is a real shared validator that justifies that location. Generic JSON
|
|
||||||
syntax and JSON schema validators are likely candidates for
|
|
||||||
`internal/validators/generic/valid_json` and
|
|
||||||
`internal/validators/generic/valid_json_schema`.
|
|
||||||
|
|
||||||
Each validator package should expose:
|
|
||||||
|
|
||||||
- a stable validator key;
|
|
||||||
- execution-class metadata;
|
|
||||||
- a constructor;
|
|
||||||
- a validator spec suitable for registration;
|
|
||||||
- a `Register` function;
|
|
||||||
- focused tests for decisions, warnings, errors, and diagnostics behavior.
|
|
||||||
|
|
||||||
Reusable validator runtime mechanics should live in framework code, such as
|
|
||||||
`internal/framework/validators`, not in concrete validator packages. This package
|
|
||||||
can own shared helpers for decision cardinality, approval/rejection construction,
|
|
||||||
LLM validator batching, validator diagnostics, and Scriptorium request plumbing.
|
|
||||||
|
|
||||||
The concrete validator packages should own policy: what they inspect, what they
|
|
||||||
approve or reject, what warning reason codes they emit, and how they interpret
|
|
||||||
domain-specific data.
|
|
||||||
|
|
||||||
## Validator Design Policy
|
|
||||||
|
|
||||||
Validators should follow a small-tool model: each validator should do one thing
|
|
||||||
well. If a validator both rejects output and emits unrelated warnings, split
|
|
||||||
those concerns into separate validators so production mappings can include,
|
|
||||||
exclude, and order them independently.
|
|
||||||
|
|
||||||
Validators are read-only. A validator must not mutate pipeline state, rewrite
|
|
||||||
module output, materialize raw output into typed stage output, or enrich the
|
|
||||||
`ModuleOutput` passed to later validators. A validator returns an
|
|
||||||
accept/reject verdict for the output it evaluates, plus any warnings or
|
|
||||||
diagnostic references. Any conversion from raw module output into a downstream
|
|
||||||
representation is a separate materialization concern and must not be hidden
|
|
||||||
inside a validator.
|
|
||||||
|
|
||||||
The initial generic validator set should include:
|
|
||||||
|
|
||||||
- `generic/always_accept`: accepts returned module output unchanged. This is
|
|
||||||
functionally equivalent to a no-op validator and is primarily useful for tests,
|
|
||||||
demonstrations, and explicit pass-through configurations.
|
|
||||||
- `generic/always_reject`: rejects returned module output without inspecting it.
|
|
||||||
This is primarily useful for tests and for proving rejection plumbing,
|
|
||||||
manifests, and diagnostics.
|
|
||||||
- `generic/valid_json`: inspects raw returned module output and accepts only
|
|
||||||
syntactically valid JSON.
|
|
||||||
- `generic/valid_json_schema`: compares raw returned JSON with the module's
|
|
||||||
configured response schema and accepts only schema-conformant output.
|
|
||||||
|
|
||||||
The exact keys may be adjusted during implementation to match local naming
|
|
||||||
conventions, but the validator set should preserve these four behaviors.
|
|
||||||
|
|
||||||
For the current D&D spell behavior, the target split is:
|
|
||||||
|
|
||||||
- `generic/valid_json`: rejects returned module output that is not syntactically
|
|
||||||
valid JSON.
|
|
||||||
- `generic/valid_json_schema`: rejects returned JSON that does not conform to
|
|
||||||
the configured response schema.
|
|
||||||
- `extract/dnd/spells/shape`: rejects malformed spell-cast payloads and missing
|
|
||||||
required spell fields.
|
|
||||||
- `extract/dnd/spells/source_refs`: rejects missing or invalid source
|
|
||||||
references.
|
|
||||||
- `extract/dnd/spells/source_relatedness`: warning-only validator that reports
|
|
||||||
when a spell name is not found in the cited source text.
|
|
||||||
|
|
||||||
Production default mappings should generally list deterministic validators
|
|
||||||
before LLM-backed validators. This keeps cheap structural failures from consuming
|
|
||||||
model calls and keeps diagnostics easier to interpret. Pipeline-configured order
|
|
||||||
should still be authoritative; if a user explicitly lists an LLM-backed
|
|
||||||
validator before a deterministic validator, the framework should honor that
|
|
||||||
order rather than silently reshuffling it.
|
|
||||||
|
|
||||||
## Execution Classes
|
|
||||||
|
|
||||||
Validator specs should declare an execution class:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ExecutionClass string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ExecutionClassDeterministic ExecutionClass = "deterministic"
|
|
||||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
Execution class should be metadata on the validator spec or registered
|
|
||||||
definition, not an ad hoc convention inferred from package paths. It should be
|
|
||||||
used for:
|
|
||||||
|
|
||||||
- human-readable catalog and manifest reporting;
|
|
||||||
- diagnostics and timing summaries;
|
|
||||||
- operational policy such as concurrency budgeting for LLM-backed validators;
|
|
||||||
- default mapping review, where deterministic validators should usually appear
|
|
||||||
before LLM-backed validators.
|
|
||||||
|
|
||||||
Execution class should not by itself imply compatibility with a stage or module.
|
|
||||||
|
|
||||||
## Validator Contract
|
|
||||||
|
|
||||||
The validator framework should support validation of outputs from `chunk`,
|
|
||||||
`extract`, `merge`, and `normalize` stages. The contract should be generalized
|
|
||||||
enough for stage-specific validators to inspect the output they care about while
|
|
||||||
ignoring irrelevant fields.
|
|
||||||
|
|
||||||
The current `contracts.RawValidationRequest` is the right starting point. It
|
|
||||||
already carries stage, lane, module, source, source and chunk provenance,
|
|
||||||
response schema metadata, raw payload, and run metadata. The final contract
|
|
||||||
should evolve from that raw-output shape rather than from the legacy
|
|
||||||
artifact-candidate `ValidationRequest`.
|
|
||||||
|
|
||||||
Additional fields needed for the full validator system include:
|
|
||||||
|
|
||||||
- source input material when a validator needs to compare module output to the
|
|
||||||
original source payload;
|
|
||||||
- session ID;
|
|
||||||
- resolved references for the validated target;
|
|
||||||
- LLM client and profile for LLM-backed validators;
|
|
||||||
- validator options;
|
|
||||||
- chunk output collections when validating a chunk module;
|
|
||||||
- ordered upstream output envelopes when validating merge or normalize behavior.
|
|
||||||
|
|
||||||
A shared module-output envelope should represent the validation boundary.
|
|
||||||
Validators may inspect raw returned content and any already-existing typed stage
|
|
||||||
envelope, such as `SourceChunk` values for chunk validation, but they must not
|
|
||||||
modify it.
|
|
||||||
|
|
||||||
Conceptually:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ModuleOutput struct {
|
|
||||||
Stage pipeline.ModuleStage
|
|
||||||
ModuleKey string
|
|
||||||
LaneID string
|
|
||||||
|
|
||||||
RawContent []byte
|
|
||||||
MediaType string
|
|
||||||
ResponseSchema contracts.ResponseSchema
|
|
||||||
|
|
||||||
SourceID string
|
|
||||||
ChunkID string
|
|
||||||
ChunkIndex int
|
|
||||||
Chunks []contracts.SourceChunk
|
|
||||||
Warnings []contracts.Warning
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The final implementation does not need to use this exact shape, but it should
|
|
||||||
preserve the boundary: returned raw module output enters validation as immutable
|
|
||||||
module output. Validators may parse raw bytes internally to decide approve,
|
|
||||||
reject, or warn, but parsing inside a validator must not create or replace the
|
|
||||||
payload passed to later stages.
|
|
||||||
|
|
||||||
The result should continue to express validator identity, warnings, and explicit
|
|
||||||
decisions. For output collections, the implementation should define an explicit
|
|
||||||
decision shape rather than silently mutating lists. Validator decisions reject or
|
|
||||||
approve output; validators do not rewrite output.
|
|
||||||
|
|
||||||
An empty validator list is always valid. With no validators, the framework should
|
|
||||||
pass module output forward unchanged and treat the output as approved for that
|
|
||||||
validation point. If the approved output cannot be consumed by a later stage
|
|
||||||
because its media type or envelope shape is unsuitable, that failure should be
|
|
||||||
reported at the downstream boundary that requires a different shape, not as an
|
|
||||||
implicit pre-validation rejection.
|
|
||||||
|
|
||||||
## Module Development Workflow
|
|
||||||
|
|
||||||
The validation system should make iterative module development easier. Once
|
|
||||||
pipeline overrides are implemented, a module author should be able to start with
|
|
||||||
an explicit empty validator mapping and inspect returned raw LLM output without
|
|
||||||
first satisfying JSON syntax, schema, media-type, or domain validators.
|
|
||||||
|
|
||||||
A typical development path should be:
|
|
||||||
|
|
||||||
1. Configure an empty validator set for the module and inspect raw returned
|
|
||||||
output.
|
|
||||||
2. Add `generic/valid_json` and adjust prompts until the model reliably returns
|
|
||||||
syntactically valid JSON.
|
|
||||||
3. Add `generic/valid_json_schema` and iterate on prompt/schema alignment.
|
|
||||||
4. Add media-type or schema validators appropriate to the module's intended
|
|
||||||
output format.
|
|
||||||
5. Add domain-specific validators one at a time until production policy is
|
|
||||||
represented explicitly in the chain.
|
|
||||||
|
|
||||||
This workflow is a central reason for making output validation explicit and
|
|
||||||
composable rather than hiding schema, shape, or domain checks inside module
|
|
||||||
implementation code.
|
|
||||||
|
|
||||||
## Central Production Mappings
|
|
||||||
|
|
||||||
Production defaults should be defined in a central, human-readable location near
|
|
||||||
the production module and validator registries. The mapping should be keyed by
|
|
||||||
stage and module key, not only by module key, so future modules can share keys
|
|
||||||
only when stage context makes their ownership unambiguous.
|
|
||||||
|
|
||||||
Conceptually:
|
|
||||||
|
|
||||||
```go
|
|
||||||
{
|
|
||||||
Stage: pipeline.StageExtract,
|
|
||||||
Module: "dnd/spells",
|
|
||||||
Validators: []pipeline.ModuleBinding{
|
|
||||||
pipeline.Binding("generic/valid_json"),
|
|
||||||
pipeline.Binding("generic/valid_json_schema"),
|
|
||||||
pipeline.Binding("extract/dnd/spells/shape"),
|
|
||||||
pipeline.Binding("extract/dnd/spells/source_refs"),
|
|
||||||
pipeline.Binding("extract/dnd/spells/source_relatedness"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The production catalog should expose three related surfaces together:
|
|
||||||
|
|
||||||
- available modules;
|
|
||||||
- available validators;
|
|
||||||
- default module-to-validator mappings.
|
|
||||||
|
|
||||||
This makes production validation policy reviewable without constructing concrete
|
|
||||||
modules or searching inside module implementation packages.
|
|
||||||
|
|
||||||
The validator registry should expose registered validator specs without building
|
|
||||||
validators, including key and execution class. Building a validator should still
|
|
||||||
be available for runtime execution.
|
|
||||||
|
|
||||||
The mapping surface should preserve the current useful behavior of stage/module
|
|
||||||
lookup and empty-chain approval while adding validator specs, execution-class
|
|
||||||
metadata, production registration, config override integration, and manifest
|
|
||||||
reporting of the resolved chain.
|
|
||||||
|
|
||||||
## Pipeline Overrides
|
|
||||||
|
|
||||||
Pipeline configuration should be able to override the central default mapping
|
|
||||||
for a module binding. Current configuration validation rejects non-empty
|
|
||||||
validator lists, and the current config/profile structs do not preserve whether
|
|
||||||
an empty list was explicitly configured or simply omitted. The target config
|
|
||||||
model must preserve that distinction.
|
|
||||||
|
|
||||||
Override semantics should distinguish three states:
|
|
||||||
|
|
||||||
- unset validators: use the central production/default mapping;
|
|
||||||
- explicit empty validators: run no validators and pass output forward;
|
|
||||||
- explicit non-empty validators: run exactly the configured validators in the
|
|
||||||
configured order.
|
|
||||||
|
|
||||||
This keeps the happy path concise while preserving advanced control for
|
|
||||||
experimentation, debugging, and custom deployments.
|
|
||||||
|
|
||||||
The run manifest should record the resolved validator chain for each validation
|
|
||||||
point so completed runs remain auditable after defaults or configuration change.
|
|
||||||
Each manifest entry should include at least validator key and execution class,
|
|
||||||
and should preserve the resolved order actually used for the run.
|
|
||||||
|
|
||||||
## LLM-Backed Validator Runtime
|
|
||||||
|
|
||||||
LLM-backed validators should use the same validator contract as deterministic
|
|
||||||
validators. Shared framework runtime should provide common support for:
|
|
||||||
|
|
||||||
- Scriptorium request construction;
|
|
||||||
- validator prompt and schema provenance;
|
|
||||||
- diagnostics redaction;
|
|
||||||
- optional batching or context-window controls when validator inputs are large;
|
|
||||||
- mapping successful LLM validator responses into validator decisions and
|
|
||||||
warnings;
|
|
||||||
- consistent handling of Scriptorium/runtime errors.
|
|
||||||
|
|
||||||
Scriptorium errors raised during validator execution should be treated as
|
|
||||||
validator execution errors unless a specific validator deliberately converts a
|
|
||||||
successful response into reject/warn decisions. This keeps provider/runtime
|
|
||||||
failure distinct from a validator's semantic rejection of module output.
|
|
||||||
|
|
||||||
## Stage Coverage
|
|
||||||
|
|
||||||
Validators should be composable across all LLM-eligible stages:
|
|
||||||
|
|
||||||
- `chunk`: validators can evaluate chunk boundaries, coverage, overlap, metadata,
|
|
||||||
or module-specific chunk quality.
|
|
||||||
- `extract`: validators can evaluate raw extracted output, source references,
|
|
||||||
payload shape, evidence quality, media type, or domain constraints.
|
|
||||||
- `merge`: validators can evaluate merged output, cross-chunk consistency,
|
|
||||||
deduplication results, media type, or domain-specific reconciliation.
|
|
||||||
- `normalize`: validators can evaluate normalized output, final shape,
|
|
||||||
post-processing results, media type, or domain-specific policy.
|
|
||||||
|
|
||||||
The framework should not require compatibility declarations in this pass. A
|
|
||||||
validator mapped to an unsuitable output shape should return a clear error, or
|
|
||||||
approve unchanged only when that is explicitly the validator's documented
|
|
||||||
behavior.
|
|
||||||
|
|
||||||
## Documentation Impact
|
|
||||||
|
|
||||||
Current-behavior docs and policy should describe the implemented validation
|
|
||||||
system once the refactor is complete:
|
|
||||||
|
|
||||||
- `docs/policy/architecture.md` should describe centralized validator mappings
|
|
||||||
rather than module-owned validator chains.
|
|
||||||
- `docs/internal/modules.md` should remove claims that concrete modules own
|
|
||||||
validator defaults.
|
|
||||||
- Internal validation docs should describe validator package ownership, mapping
|
|
||||||
precedence, empty-chain approval behavior, and LLM-backed validator support.
|
|
||||||
- User/config docs should replace the current "configured validators are
|
|
||||||
reserved and rejected" language with the implemented pipeline override
|
|
||||||
contract.
|
|
||||||
|
|
||||||
Roadmap docs should not remain the canonical description of implemented
|
|
||||||
validation behavior after the refactor is complete.
|
|
||||||
|
|||||||
@@ -118,9 +118,10 @@ Symptoms include:
|
|||||||
|
|
||||||
Fix:
|
Fix:
|
||||||
|
|
||||||
- Confirm the selected chunker, extractor, merger, or normalizer declares the slot. The
|
- Confirm the selected chunker, extractor, merger, or normalizer declares the
|
||||||
implemented `dnd/scenes` chunker and `dnd/spells` extractor declare optional
|
slot. The implemented `dnd/scenes` chunker and `dnd/spells` extractor declare
|
||||||
`roster` and `glossary` slots.
|
optional `players`, `party`, and `glossary` slots, plus `roster` as a
|
||||||
|
deprecated compatibility alias for `party`.
|
||||||
- Use a specific selector when more than one selected target declares the same
|
- Use a specific selector when more than one selected target declares the same
|
||||||
slot: `chunk.context=./context.txt`,
|
slot: `chunk.context=./context.txt`,
|
||||||
`spells.extract.context=./extract-context.txt`,
|
`spells.extract.context=./extract-context.txt`,
|
||||||
@@ -317,11 +318,31 @@ Explanation and fixes:
|
|||||||
pass to the next pipeline stage.
|
pass to the next pipeline stage.
|
||||||
- Check `rejected.json` for the stage, lane, module, chunk, validator, reason,
|
- Check `rejected.json` for the stage, lane, module, chunk, validator, reason,
|
||||||
message, and attempt count.
|
message, and attempt count.
|
||||||
|
- Check `manifest.json` `validator_chains` to see the exact resolved validators
|
||||||
|
and order used for the rejected validation point. The production `dnd/spells`
|
||||||
|
extractor runs JSON syntax, JSON schema, D&D spell shape, source-reference,
|
||||||
|
and source-relatedness validators by default.
|
||||||
|
- If the configured chain is not what you expected, inspect the selected
|
||||||
|
binding in config. Omitted `validators` uses production defaults,
|
||||||
|
`validators: []` disables validators for that binding, and a non-empty list
|
||||||
|
replaces the default chain in configured order.
|
||||||
|
- Run `notarius config validate --pipeline <id>` to catch unknown validator keys
|
||||||
|
and invalid validator `llm_profile` usage before running the pipeline.
|
||||||
- Increase a module binding's `retries` only when re-running the same module
|
- Increase a module binding's `retries` only when re-running the same module
|
||||||
input can reasonably produce an acceptable output.
|
input can reasonably produce an acceptable output.
|
||||||
- If rejection is deterministic, fix the source input, module configuration, or
|
- If rejection is deterministic, fix the source input, module configuration, or
|
||||||
validation policy rather than adding retries.
|
validation policy rather than adding retries.
|
||||||
|
|
||||||
|
Common production D&D spell validator reasons:
|
||||||
|
|
||||||
|
- `invalid_json`: the raw output is not valid JSON.
|
||||||
|
- `json_schema_invalid`: the raw JSON does not match the spell response schema.
|
||||||
|
- `invalid_spell_shape`: required spell-cast fields are missing or malformed.
|
||||||
|
- `invalid_source_refs`: source references are missing or do not point to valid
|
||||||
|
source units.
|
||||||
|
- `spell_not_near_source`: warning-only; the spell name was not found near the
|
||||||
|
cited source text.
|
||||||
|
|
||||||
## Retry Exhaustion
|
## Retry Exhaustion
|
||||||
|
|
||||||
Symptoms include:
|
Symptoms include:
|
||||||
|
|||||||
6
go.mod
6
go.mod
@@ -4,10 +4,8 @@ go 1.25.5
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0
|
gitea.maximumdirect.net/eric/scriptorium v0.11.0
|
||||||
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require golang.org/x/text v0.14.0 // indirect
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
|
||||||
golang.org/x/text v0.14.0 // indirect
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
||||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
|
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
|
||||||
|
spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape"
|
||||||
|
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs"
|
||||||
|
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness"
|
||||||
|
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_accept"
|
||||||
|
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_reject"
|
||||||
|
validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json"
|
||||||
|
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
func productionRegistries() (pipeline.Registries, error) {
|
func productionRegistries() (pipeline.Registries, error) {
|
||||||
@@ -26,6 +33,7 @@ func productionRegistries() (pipeline.Registries, error) {
|
|||||||
Mergers: pipeline.NewMergerRegistry(),
|
Mergers: pipeline.NewMergerRegistry(),
|
||||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||||
Validators: pipeline.NewValidatorRegistry(),
|
Validators: pipeline.NewValidatorRegistry(),
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := seriatim.Register(registries.Inputs); err != nil {
|
if err := seriatim.Register(registries.Inputs); err != nil {
|
||||||
@@ -46,12 +54,56 @@ func productionRegistries() (pipeline.Registries, error) {
|
|||||||
if err := noop.Register(registries.Normalizers); err != nil {
|
if err := noop.Register(registries.Normalizers); err != nil {
|
||||||
return pipeline.Registries{}, fmt.Errorf("register noop normalizer: %w", err)
|
return pipeline.Registries{}, fmt.Errorf("register noop normalizer: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := registerProductionValidators(registries.Validators); err != nil {
|
||||||
|
return pipeline.Registries{}, err
|
||||||
|
}
|
||||||
|
if err := registerProductionValidatorChains(registries.ValidatorChains); err != nil {
|
||||||
|
return pipeline.Registries{}, err
|
||||||
|
}
|
||||||
if err := jsonoutput.Register(registries.Outputs); err != nil {
|
if err := jsonoutput.Register(registries.Outputs); err != nil {
|
||||||
return pipeline.Registries{}, fmt.Errorf("register json output encoder: %w", err)
|
return pipeline.Registries{}, fmt.Errorf("register json output encoder: %w", err)
|
||||||
}
|
}
|
||||||
return registries, nil
|
return registries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func registerProductionValidators(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
registrations := []struct {
|
||||||
|
name string
|
||||||
|
register func(*pipeline.ValidatorRegistry) error
|
||||||
|
}{
|
||||||
|
{name: "generic always accept validator", register: alwaysaccept.Register},
|
||||||
|
{name: "generic always reject validator", register: alwaysreject.Register},
|
||||||
|
{name: "generic valid json validator", register: validjson.Register},
|
||||||
|
{name: "generic valid json schema validator", register: validjsonschema.Register},
|
||||||
|
{name: "dnd spell shape validator", register: spellshape.Register},
|
||||||
|
{name: "dnd spell source references validator", register: spellsourcerefs.Register},
|
||||||
|
{name: "dnd spell source relatedness validator", register: spellrelatedness.Register},
|
||||||
|
}
|
||||||
|
for _, registration := range registrations {
|
||||||
|
if err := registration.register(registry); err != nil {
|
||||||
|
return fmt.Errorf("register %s: %w", registration.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerProductionValidatorChains(registry *pipeline.ValidatorChainRegistry) error {
|
||||||
|
if err := registry.Register(pipeline.ValidatorChainMapping{
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Module: spells.Key,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding(validjson.Key),
|
||||||
|
pipeline.Binding(validjsonschema.Key),
|
||||||
|
pipeline.Binding(spellshape.Key),
|
||||||
|
pipeline.Binding(spellsourcerefs.Key),
|
||||||
|
pipeline.Binding(spellrelatedness.Key),
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("register dnd spells validator chain: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func productionCatalog() (pipeline.ModuleCatalog, error) {
|
func productionCatalog() (pipeline.ModuleCatalog, error) {
|
||||||
registries, err := productionRegistries()
|
registries, err := productionRegistries()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -99,6 +151,7 @@ func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalo
|
|||||||
Mergers: registries.Mergers,
|
Mergers: registries.Mergers,
|
||||||
Normalizers: registries.Normalizers,
|
Normalizers: registries.Normalizers,
|
||||||
Validators: registries.Validators,
|
Validators: registries.Validators,
|
||||||
|
ValidatorChains: registries.ValidatorChains,
|
||||||
Outputs: registries.Outputs,
|
Outputs: registries.Outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,6 +164,7 @@ func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
|
|||||||
Mergers: catalog.Mergers,
|
Mergers: catalog.Mergers,
|
||||||
Normalizers: catalog.Normalizers,
|
Normalizers: catalog.Normalizers,
|
||||||
Validators: catalog.Validators,
|
Validators: catalog.Validators,
|
||||||
|
ValidatorChains: catalog.ValidatorChains,
|
||||||
Outputs: catalog.Outputs,
|
Outputs: catalog.Outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -122,6 +176,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
|
|||||||
catalog.Mergers == nil &&
|
catalog.Mergers == nil &&
|
||||||
catalog.Normalizers == nil &&
|
catalog.Normalizers == nil &&
|
||||||
catalog.Validators == nil &&
|
catalog.Validators == nil &&
|
||||||
|
catalog.ValidatorChains == nil &&
|
||||||
catalog.Outputs == nil
|
catalog.Outputs == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,6 +187,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
|||||||
registries.Mergers == nil &&
|
registries.Mergers == nil &&
|
||||||
registries.Normalizers == nil &&
|
registries.Normalizers == nil &&
|
||||||
registries.Validators == nil &&
|
registries.Validators == nil &&
|
||||||
|
registries.ValidatorChains == nil &&
|
||||||
registries.Outputs == nil
|
registries.Outputs == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -495,6 +495,13 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
|
|||||||
add(lane.Merge)
|
add(lane.Merge)
|
||||||
add(lane.Normalize)
|
add(lane.Normalize)
|
||||||
}
|
}
|
||||||
|
for _, chain := range resolved.ValidatorChains {
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||||
|
add(validator.Binding)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
ids := make([]string, 0, len(seen))
|
ids := make([]string, 0, len(seen))
|
||||||
for id := range seen {
|
for id := range seen {
|
||||||
ids = append(ids, id)
|
ids = append(ids, id)
|
||||||
|
|||||||
@@ -27,6 +27,13 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
||||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
|
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
|
||||||
|
spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape"
|
||||||
|
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs"
|
||||||
|
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness"
|
||||||
|
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_accept"
|
||||||
|
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_reject"
|
||||||
|
validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json"
|
||||||
|
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/scriptorium"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -121,13 +128,13 @@ func TestRunConfigValidateSuccessWithFakeCatalog(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
|
func TestProductionCatalogIncludesProductionModulesValidatorsAndDefaults(t *testing.T) {
|
||||||
catalog, err := productionCatalog()
|
catalog, err := productionCatalog()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("productionCatalog() error = %v, want nil", err)
|
t.Fatalf("productionCatalog() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tests := []struct {
|
moduleTests := []struct {
|
||||||
name string
|
name string
|
||||||
got func() (pipeline.ModuleSpec, bool)
|
got func() (pipeline.ModuleSpec, bool)
|
||||||
want pipeline.ModuleSpec
|
want pipeline.ModuleSpec
|
||||||
@@ -169,7 +176,7 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range moduleTests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
got, ok := test.got()
|
got, ok := test.got()
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -180,6 +187,42 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validatorTests := []pipeline.ValidatorSpec{
|
||||||
|
alwaysaccept.Spec(),
|
||||||
|
alwaysreject.Spec(),
|
||||||
|
validjson.Spec(),
|
||||||
|
validjsonschema.Spec(),
|
||||||
|
spellshape.Spec(),
|
||||||
|
spellsourcerefs.Spec(),
|
||||||
|
spellrelatedness.Spec(),
|
||||||
|
}
|
||||||
|
for _, want := range validatorTests {
|
||||||
|
t.Run("validator "+want.Key, func(t *testing.T) {
|
||||||
|
got, ok := catalog.Validators.Spec(want.Key)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("validator spec %q ok = false, want true", want.Key)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("validator spec = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
gotChain := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key)
|
||||||
|
wantChain := []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding(validjson.Key),
|
||||||
|
pipeline.Binding(validjsonschema.Key),
|
||||||
|
pipeline.Binding(spellshape.Key),
|
||||||
|
pipeline.Binding(spellsourcerefs.Key),
|
||||||
|
pipeline.Binding(spellrelatedness.Key),
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotChain, wantChain) {
|
||||||
|
t.Fatalf("dnd spell default validator chain = %#v, want %#v", gotChain, wantChain)
|
||||||
|
}
|
||||||
|
if got := catalog.ValidatorChains.Validators(pipeline.StageChunk, generic.Key); len(got) != 0 {
|
||||||
|
t.Fatalf("generic chunker default validator chain = %#v, want empty", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProductionPromptAssetsRegisterAndPrepareDndPrompts(t *testing.T) {
|
func TestProductionPromptAssetsRegisterAndPrepareDndPrompts(t *testing.T) {
|
||||||
@@ -336,6 +379,50 @@ func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateRejectsUnknownProductionValidator(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - missing/validator\n"))
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "unknown validator") || !strings.Contains(got, "missing/validator") {
|
||||||
|
t.Fatalf("stderr = %q, want unknown validator", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateRejectsLLMProfileForDeterministicProductionValidator(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, `version: 2
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
input: seriatim
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract:
|
||||||
|
module: dnd/spells
|
||||||
|
validators:
|
||||||
|
- module: generic/valid_json
|
||||||
|
llm_profile: review
|
||||||
|
`)
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
got := stderr.String()
|
||||||
|
for _, want := range []string{"llm_profile", "deterministic", validjson.Key} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("stderr = %q, want substring %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunConfigValidateReportsParseErrors(t *testing.T) {
|
func TestRunConfigValidateReportsParseErrors(t *testing.T) {
|
||||||
configPath := writeFile(t, "config.yml", "version: 1\n")
|
configPath := writeFile(t, "config.yml", "version: 1\n")
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
@@ -743,7 +830,7 @@ func TestRunPipelineLLMFactoryFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunPipelineCarriesInvalidLLMSourceRefsAsRawOutput(t *testing.T) {
|
func TestRunPipelineDefaultDNDSpellValidatorsRejectInvalidSourceRefs(t *testing.T) {
|
||||||
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
|
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
|
||||||
inputPath := writeSeriatimInput(t)
|
inputPath := writeSeriatimInput(t)
|
||||||
outputDir := t.TempDir()
|
outputDir := t.TempDir()
|
||||||
@@ -756,11 +843,107 @@ func TestRunPipelineCarriesInvalidLLMSourceRefsAsRawOutput(t *testing.T) {
|
|||||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "outputs=0") || !strings.Contains(stdout.String(), "rejected=1") {
|
||||||
|
t.Fatalf("stdout = %q, want rejected output count", stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest)
|
||||||
|
if manifest.ValidationStatus != "rejected" {
|
||||||
|
t.Fatalf("validation status = %q, want rejected", manifest.ValidationStatus)
|
||||||
|
}
|
||||||
|
if len(manifest.RejectedOutputs) != 1 {
|
||||||
|
t.Fatalf("rejected outputs = %#v, want one rejection", manifest.RejectedOutputs)
|
||||||
|
}
|
||||||
|
rejection := manifest.RejectedOutputs[0]
|
||||||
|
if rejection.ValidatorName != spellsourcerefs.Key || rejection.ReasonCode != spellsourcerefs.ReasonCode {
|
||||||
|
t.Fatalf("rejection = %#v, want source reference validator rejection", rejection)
|
||||||
|
}
|
||||||
|
gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)
|
||||||
|
wantKeys := []string{validjson.Key, validjsonschema.Key, spellshape.Key, spellsourcerefs.Key, spellrelatedness.Key}
|
||||||
|
if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, wantKeys) {
|
||||||
|
t.Fatalf("validator chain keys = %#v, want %#v", got, wantKeys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineExplicitEmptyValidatorOverrideDisablesDNDSpellDefaults(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", " []\n"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
outputDir := t.TempDir()
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
client := newFakeRunLLMClient(true)
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
|
})
|
||||||
|
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") {
|
if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") {
|
||||||
t.Fatalf("stdout = %q, want raw output count", stdout.String())
|
t.Fatalf("stdout = %q, want accepted output count", stdout.String())
|
||||||
|
}
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest)
|
||||||
|
gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)
|
||||||
|
if len(gotChain.Validators) != 0 {
|
||||||
|
t.Fatalf("validator chain = %#v, want explicit empty chain", gotChain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineExplicitValidatorOverrideReplacesDNDSpellDefaults(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - "+alwaysaccept.Key+"\n"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
outputDir := t.TempDir()
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
client := newFakeRunLLMClient(true)
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") {
|
||||||
|
t.Fatalf("stdout = %q, want accepted output count", stdout.String())
|
||||||
|
}
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest)
|
||||||
|
gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)
|
||||||
|
if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, []string{alwaysaccept.Key}) {
|
||||||
|
t.Fatalf("validator chain keys = %#v, want explicit override", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineConfiguredValidatorOrderIsPreserved(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - "+alwaysaccept.Key+"\n - "+validjson.Key+"\n"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
outputDir := t.TempDir()
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest)
|
||||||
|
gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)
|
||||||
|
wantKeys := []string{alwaysaccept.Key, validjson.Key}
|
||||||
|
if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, wantKeys) {
|
||||||
|
t.Fatalf("validator chain keys = %#v, want %#v", got, wantKeys)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -816,6 +999,42 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateChecksExplicitLLMValidatorProfileIDs(t *testing.T) {
|
||||||
|
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
|
||||||
|
configPath := writeTestConfig(t, `version: 2
|
||||||
|
scriptorium:
|
||||||
|
profile_file: `+profilePath+`
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
validators:
|
||||||
|
- module: fake/llm-validator
|
||||||
|
llm_profile: missing
|
||||||
|
`)
|
||||||
|
catalog := fakeCatalog(t)
|
||||||
|
mustRegisterValidator(t, catalog.Validators, pipeline.ValidatorSpec{
|
||||||
|
Key: "fake/llm-validator",
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{
|
||||||
|
Catalog: catalog,
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") {
|
||||||
|
t.Fatalf("stderr = %q, want unknown validator Scriptorium profile", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunConfigValidateIncludesMergeAndIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) {
|
func TestRunConfigValidateIncludesMergeAndIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) {
|
||||||
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
|
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
|
||||||
configPath := writeTestConfig(t, `version: 2
|
configPath := writeTestConfig(t, `version: 2
|
||||||
@@ -861,13 +1080,29 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
|
|||||||
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
|
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
|
||||||
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
|
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
|
||||||
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
|
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
|
||||||
Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}},
|
},
|
||||||
|
},
|
||||||
|
ValidatorChains: []pipeline.ResolvedValidatorChain{
|
||||||
|
{
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
LaneID: "events",
|
||||||
|
ModuleKey: "extract",
|
||||||
|
Validators: []pipeline.ResolvedValidator{
|
||||||
|
{
|
||||||
|
Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"},
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"},
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
got := effectiveLLMProfileIDs(resolved)
|
got := effectiveLLMProfileIDs(resolved)
|
||||||
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"}
|
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
|
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
@@ -2254,18 +2489,18 @@ func TestExampleFixtureFailureCoverage(t *testing.T) {
|
|||||||
wantStderr: "completion unavailable",
|
wantStderr: "completion unavailable",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "malformed LLM response carried as raw output",
|
name: "malformed LLM response rejected by validation",
|
||||||
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
|
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
|
||||||
factory: fakeLLMFactory(newMalformedRunLLMClient(), nil),
|
factory: fakeLLMFactory(newMalformedRunLLMClient(), nil),
|
||||||
wantCode: 0,
|
wantCode: 0,
|
||||||
wantOutputStatus: "approved",
|
wantOutputStatus: "rejected",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid source reference raw output",
|
name: "invalid source reference rejected by validation",
|
||||||
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
|
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
|
||||||
factory: fakeLLMFactory(newFakeRunLLMClient(true), nil),
|
factory: fakeLLMFactory(newFakeRunLLMClient(true), nil),
|
||||||
wantCode: 0,
|
wantCode: 0,
|
||||||
wantOutputStatus: "approved",
|
wantOutputStatus: "rejected",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2424,6 +2659,18 @@ pipelines:
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mvpConfigYAMLWithExtractValidators(pipelineID string, validators string) string {
|
||||||
|
return `version: 2
|
||||||
|
pipelines:
|
||||||
|
` + pipelineID + `:
|
||||||
|
input: seriatim
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract:
|
||||||
|
module: dnd/spells
|
||||||
|
validators:` + validators
|
||||||
|
}
|
||||||
|
|
||||||
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
|
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
|
||||||
return `version: 2
|
return `version: 2
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -2886,6 +3133,25 @@ func resolvedArtifactLane(t *testing.T, resolved pipeline.ResolvedPipeline, lane
|
|||||||
return pipeline.ResolvedArtifactLane{}
|
return pipeline.ResolvedArtifactLane{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func manifestValidatorChain(t *testing.T, manifest artifacts.RunManifest, stage pipeline.ModuleStage, laneID string, module string) artifacts.ValidatorChainManifest {
|
||||||
|
t.Helper()
|
||||||
|
for _, chain := range manifest.ValidatorChains {
|
||||||
|
if chain.Stage == string(stage) && chain.LaneID == laneID && chain.ModuleKey == module {
|
||||||
|
return chain
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("validator chain %s/%s/%s not found in %#v", stage, laneID, module, manifest.ValidatorChains)
|
||||||
|
return artifacts.ValidatorChainManifest{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestValidatorKeys(chain artifacts.ValidatorChainManifest) []string {
|
||||||
|
keys := make([]string, 0, len(chain.Validators))
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
keys = append(keys, validator.Key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
func assertNoTemporaryFiles(t *testing.T, root string) {
|
func assertNoTemporaryFiles(t *testing.T, root string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||||
@@ -2937,6 +3203,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
Validators: validators,
|
Validators: validators,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2983,6 +3250,32 @@ func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ValidatorSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) {
|
||||||
|
return fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeConfigValidator struct {
|
||||||
|
name string
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator fakeConfigValidator) Name() string {
|
||||||
|
return validator.name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator fakeConfigValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return validator.executionClass
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator fakeConfigValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||||
return func(key string) (string, bool) {
|
return func(key string) (string, bool) {
|
||||||
value, ok := values[key]
|
value, ok := values[key]
|
||||||
|
|||||||
@@ -1,47 +1,29 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ArtifactCandidate struct {
|
|
||||||
Index int `json:"index"`
|
|
||||||
ExtractorKey string `json:"extractor_key"`
|
|
||||||
ArtifactType string `json:"artifact_type"`
|
|
||||||
SchemaVersion string `json:"schema_version"`
|
|
||||||
Payload json.RawMessage `json:"payload"`
|
|
||||||
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Artifact struct {
|
|
||||||
ExtractorKey string `json:"extractor_key"`
|
|
||||||
ArtifactType string `json:"artifact_type"`
|
|
||||||
SchemaVersion string `json:"schema_version"`
|
|
||||||
Payload json.RawMessage `json:"payload"`
|
|
||||||
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RejectedArtifact struct {
|
|
||||||
Candidate ArtifactCandidate `json:"candidate"`
|
|
||||||
ValidatorName string `json:"validator_name"`
|
|
||||||
ReasonCode string `json:"reason_code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ArtifactLaneManifest struct {
|
type ArtifactLaneManifest struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Extractor string `json:"extractor"`
|
Extractor string `json:"extractor"`
|
||||||
Merger string `json:"merger"`
|
Merger string `json:"merger"`
|
||||||
Normalizer string `json:"normalizer"`
|
Normalizer string `json:"normalizer"`
|
||||||
Validators []string `json:"validators,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ValidatorChainManifest struct {
|
||||||
|
Stage string `json:"stage"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
Validators []ValidatorManifest `json:"validators"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorManifest struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
ExecutionClass string `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type LLMProfileManifest struct {
|
type LLMProfileManifest struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Provider string `json:"provider,omitempty"`
|
Provider string `json:"provider,omitempty"`
|
||||||
@@ -100,6 +82,7 @@ type RunManifest struct {
|
|||||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||||
|
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
|
||||||
References []ReferenceProvenance `json:"references,omitempty"`
|
References []ReferenceProvenance `json:"references,omitempty"`
|
||||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||||
@@ -110,26 +93,3 @@ type RunManifest struct {
|
|||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
|
||||||
return Artifact{
|
|
||||||
ExtractorKey: candidate.ExtractorKey,
|
|
||||||
ArtifactType: candidate.ArtifactType,
|
|
||||||
SchemaVersion: candidate.SchemaVersion,
|
|
||||||
Payload: append(json.RawMessage(nil), candidate.Payload...),
|
|
||||||
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
|
|
||||||
Metadata: copyMetadata(candidate.Metadata),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyMetadata(metadata map[string]any) map[string]any {
|
|
||||||
if len(metadata) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
copied := make(map[string]any, len(metadata))
|
|
||||||
for key, value := range metadata {
|
|
||||||
copied[key] = value
|
|
||||||
}
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,116 +2,9 @@ package artifacts
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"reflect"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
|
||||||
candidate := ArtifactCandidate{
|
|
||||||
Index: 7,
|
|
||||||
ExtractorKey: "generic-extractor",
|
|
||||||
ArtifactType: "generic-artifact",
|
|
||||||
SchemaVersion: "v1",
|
|
||||||
Payload: json.RawMessage(`{"name":"example"}`),
|
|
||||||
SourceRefs: []source.SourceRef{
|
|
||||||
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 2},
|
|
||||||
},
|
|
||||||
Metadata: map[string]any{
|
|
||||||
"confidence": 0.75,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
artifact := ArtifactFromCandidate(candidate)
|
|
||||||
|
|
||||||
if artifact.ExtractorKey != candidate.ExtractorKey {
|
|
||||||
t.Fatalf("ExtractorKey = %q, want %q", artifact.ExtractorKey, candidate.ExtractorKey)
|
|
||||||
}
|
|
||||||
if artifact.ArtifactType != candidate.ArtifactType {
|
|
||||||
t.Fatalf("ArtifactType = %q, want %q", artifact.ArtifactType, candidate.ArtifactType)
|
|
||||||
}
|
|
||||||
if artifact.SchemaVersion != candidate.SchemaVersion {
|
|
||||||
t.Fatalf("SchemaVersion = %q, want %q", artifact.SchemaVersion, candidate.SchemaVersion)
|
|
||||||
}
|
|
||||||
if string(artifact.Payload) != string(candidate.Payload) {
|
|
||||||
t.Fatalf("Payload = %s, want %s", artifact.Payload, candidate.Payload)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(artifact.SourceRefs, candidate.SourceRefs) {
|
|
||||||
t.Fatalf("SourceRefs = %#v, want %#v", artifact.SourceRefs, candidate.SourceRefs)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(artifact.Metadata, candidate.Metadata) {
|
|
||||||
t.Fatalf("Metadata = %#v, want %#v", artifact.Metadata, candidate.Metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
candidate.Payload[0] = '['
|
|
||||||
candidate.SourceRefs[0].StartUnitID = 99
|
|
||||||
candidate.Metadata["confidence"] = 0.5
|
|
||||||
|
|
||||||
if string(artifact.Payload) != `{"name":"example"}` {
|
|
||||||
t.Fatalf("Payload changed after candidate mutation: %s", artifact.Payload)
|
|
||||||
}
|
|
||||||
if artifact.SourceRefs[0].StartUnitID != 1 {
|
|
||||||
t.Fatalf("SourceRefs changed after candidate mutation: %#v", artifact.SourceRefs)
|
|
||||||
}
|
|
||||||
if artifact.Metadata["confidence"] != 0.75 {
|
|
||||||
t.Fatalf("Metadata changed after candidate mutation: %#v", artifact.Metadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
|
||||||
candidate := ArtifactCandidate{
|
|
||||||
Index: 1,
|
|
||||||
ExtractorKey: "generic-extractor",
|
|
||||||
ArtifactType: "generic-artifact",
|
|
||||||
SchemaVersion: "v1",
|
|
||||||
Payload: json.RawMessage(`{"value":true}`),
|
|
||||||
SourceRefs: []source.SourceRef{
|
|
||||||
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
|
||||||
},
|
|
||||||
Metadata: map[string]any{
|
|
||||||
"reviewed": true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
rejected := RejectedArtifact{
|
|
||||||
Candidate: candidate,
|
|
||||||
ValidatorName: "generic-validator",
|
|
||||||
ReasonCode: "invalid",
|
|
||||||
Message: "candidate was not accepted",
|
|
||||||
}
|
|
||||||
|
|
||||||
gotJSON, err := json.Marshal(rejected)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("json.Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
|
||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertHasKeys(t, got, "candidate", "validator_name", "reason_code", "message")
|
|
||||||
|
|
||||||
gotCandidate, ok := got["candidate"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("candidate = %#v, want object", got["candidate"])
|
|
||||||
}
|
|
||||||
assertHasKeys(t, gotCandidate, "index", "extractor_key", "artifact_type", "schema_version", "payload", "source_refs", "metadata")
|
|
||||||
|
|
||||||
gotRefs, ok := gotCandidate["source_refs"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("source_refs = %#v, want array", gotCandidate["source_refs"])
|
|
||||||
}
|
|
||||||
if len(gotRefs) != 1 {
|
|
||||||
t.Fatalf("len(source_refs) = %d, want 1", len(gotRefs))
|
|
||||||
}
|
|
||||||
gotRef, ok := gotRefs[0].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("source_refs[0] = %#v, want object", gotRefs[0])
|
|
||||||
}
|
|
||||||
assertHasKeys(t, gotRef, "source_id", "start_unit_id", "end_unit_id")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
|
func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
|
||||||
gotJSON, err := json.Marshal(RunManifest{})
|
gotJSON, err := json.Marshal(RunManifest{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -136,12 +29,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
Extractor: "event-extractor",
|
Extractor: "event-extractor",
|
||||||
Merger: "appendorder",
|
Merger: "appendorder",
|
||||||
Normalizer: "noop",
|
Normalizer: "noop",
|
||||||
Validators: []string{"grounded"},
|
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
ValidatorChains: []ValidatorChainManifest{
|
||||||
|
{
|
||||||
|
Stage: "extract",
|
||||||
|
LaneID: "events",
|
||||||
|
ModuleKey: "event-extractor",
|
||||||
|
Validators: []ValidatorManifest{
|
||||||
|
{Key: "grounded", ExecutionClass: "deterministic"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
gotJSON, err := json.Marshal(manifest)
|
gotJSON, err := json.Marshal(manifest)
|
||||||
@@ -154,7 +56,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
|
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
|
||||||
|
|
||||||
profiles, ok := got["llm_profiles"].([]any)
|
profiles, ok := got["llm_profiles"].([]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -180,7 +82,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
||||||
}
|
}
|
||||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
|
||||||
|
|
||||||
|
chains, ok := got["validator_chains"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
|
||||||
|
}
|
||||||
|
if len(chains) != 1 {
|
||||||
|
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
|
||||||
|
}
|
||||||
|
chain, ok := chains[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
||||||
|
|||||||
@@ -97,6 +97,18 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
out.References = cloneStringMap(in.References)
|
out.References = cloneStringMap(in.References)
|
||||||
|
out.Validators = cloneValidatorOverride(in.Validators)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||||
|
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||||
|
if len(in.Validators) > 0 {
|
||||||
|
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||||
|
for i, binding := range in.Validators {
|
||||||
|
out.Validators[i] = cloneModuleBinding(binding)
|
||||||
|
}
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -161,6 +161,12 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
|||||||
profile.Output.LLMProfile = "output-profile"
|
profile.Output.LLMProfile = "output-profile"
|
||||||
lane := profile.Artifacts["events"]
|
lane := profile.Artifacts["events"]
|
||||||
lane.Merge.LLMProfile = "merge-profile"
|
lane.Merge.LLMProfile = "merge-profile"
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
|
||||||
|
},
|
||||||
|
}
|
||||||
profile.Artifacts["events"] = lane
|
profile.Artifacts["events"] = lane
|
||||||
cfg.Pipelines["example"] = profile
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
@@ -195,9 +201,22 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
|||||||
if eventLane.Merge.LLMProfile != "runtime" {
|
if eventLane.Merge.LLMProfile != "runtime" {
|
||||||
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
||||||
}
|
}
|
||||||
if len(eventLane.Validators) != 0 {
|
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
|
||||||
t.Fatalf("validator profiles = %#v, want none", eventLane.Validators)
|
if validatorChain == nil || len(validatorChain.Validators) != 1 {
|
||||||
|
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
|
||||||
}
|
}
|
||||||
|
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
|
||||||
|
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
|
||||||
|
for i := range chains {
|
||||||
|
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||||
|
return &chains[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ type fileModuleBinding struct {
|
|||||||
Retries int
|
Retries int
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
References map[string]string
|
References map[string]string
|
||||||
|
Validators pipeline.ValidatorOverride
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||||
@@ -102,6 +103,16 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.References = references
|
b.References = references
|
||||||
|
case "validators":
|
||||||
|
b.Validators.Set = true
|
||||||
|
var validators []fileModuleBinding
|
||||||
|
if err := valueNode.Decode(&validators); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
|
||||||
|
for i, validator := range validators {
|
||||||
|
b.Validators.Validators[i] = validator.toPipelineBinding()
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||||
}
|
}
|
||||||
@@ -119,6 +130,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
Retries: b.Retries,
|
Retries: b.Retries,
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
References: normalizedStringMap(b.References),
|
References: normalizedStringMap(b.References),
|
||||||
|
Validators: b.Validators,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -300,6 +300,61 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
|
||||||
|
cfg := parseAndApplyConfig(t, `
|
||||||
|
version: 2
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
validators: []
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
validators:
|
||||||
|
- fake/validator
|
||||||
|
- module: fake/llm-validator
|
||||||
|
llm_profile: careful
|
||||||
|
options:
|
||||||
|
threshold: 0.7
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
validators: []
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
`)
|
||||||
|
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
if !lane.Extract.Validators.Set {
|
||||||
|
t.Fatalf("extract validator override Set = false, want true")
|
||||||
|
}
|
||||||
|
validators := lane.Extract.Validators.Validators
|
||||||
|
if len(validators) != 2 {
|
||||||
|
t.Fatalf("extract validators = %#v, want two validators", validators)
|
||||||
|
}
|
||||||
|
if validators[0].Module != "fake/validator" {
|
||||||
|
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
|
||||||
|
}
|
||||||
|
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
|
||||||
|
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
|
||||||
|
}
|
||||||
|
if validators[1].Options["threshold"] != 0.7 {
|
||||||
|
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
|
||||||
|
}
|
||||||
|
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
|
||||||
|
}
|
||||||
|
if lane.Normalize.Validators.Set {
|
||||||
|
t.Fatalf("normalize validator override Set = true, want omitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
||||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||||
version: 2
|
version: 2
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
|||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
|
if len(in.ValidatorChains) > 0 {
|
||||||
|
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
|
||||||
|
for i, chain := range in.ValidatorChains {
|
||||||
|
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(in.ArtifactLanes) > 0 {
|
if len(in.ArtifactLanes) > 0 {
|
||||||
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
||||||
for i, lane := range in.ArtifactLanes {
|
for i, lane := range in.ArtifactLanes {
|
||||||
@@ -36,6 +42,20 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
|
||||||
|
out := in
|
||||||
|
if len(in.Validators) > 0 {
|
||||||
|
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
|
||||||
|
for i, validator := range in.Validators {
|
||||||
|
out.Validators[i] = pipeline.ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator.Binding),
|
||||||
|
ExecutionClass: validator.ExecutionClass,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
|
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
|
||||||
out := in
|
out := in
|
||||||
out.Extract = cloneModuleBinding(in.Extract)
|
out.Extract = cloneModuleBinding(in.Extract)
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return fmt.Errorf("pipeline %q lane %q validators are not supported by the current raw validation runner", id, laneID)
|
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -108,6 +108,9 @@ func validateBinding(
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
|
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
|
||||||
}
|
}
|
||||||
|
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if len(binding.References) == 0 {
|
if len(binding.References) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -120,6 +123,36 @@ func validateBinding(
|
|||||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||||
|
if !override.Set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch slot {
|
||||||
|
case "chunk", "extract", "merge", "normalize":
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
|
||||||
|
}
|
||||||
|
for i, validator := range override.Validators {
|
||||||
|
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||||
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
|
return fmt.Errorf("%s module must not be empty", context)
|
||||||
|
}
|
||||||
|
if len(validator.References) > 0 {
|
||||||
|
return fmt.Errorf("%s references are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.Validators.Set {
|
||||||
|
return fmt.Errorf("%s nested validators are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.Retries != 0 {
|
||||||
|
return fmt.Errorf("%s retries are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
||||||
|
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
||||||
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -341,13 +341,82 @@ func TestValidateRejectsConfiguredValidators(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Validate() error = nil, want configured validators error")
|
t.Fatal("Validate() error = nil, want configured validators error")
|
||||||
}
|
}
|
||||||
for _, want := range []string{"example", "events", "validators", "not supported"} {
|
for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
|
||||||
if !strings.Contains(err.Error(), want) {
|
if !strings.Contains(err.Error(), want) {
|
||||||
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
|
||||||
|
cfg := validConfig()
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding("fake/validator"),
|
||||||
|
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
binding pipeline.ModuleBinding
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty module",
|
||||||
|
binding: pipeline.ModuleBinding{},
|
||||||
|
want: "module must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "references",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
|
||||||
|
want: "references are not supported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested validators",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
|
||||||
|
want: "nested validators are not supported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "retries",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
|
||||||
|
want: "retries are not supported",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg := validConfig()
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{test.binding},
|
||||||
|
}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
|
err := cfg.Validate()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Validate() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func validConfig() Config {
|
func validConfig() Config {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
||||||
@@ -402,6 +471,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Requires: []string{"normalized"},
|
Requires: []string{"normalized"},
|
||||||
Provides: []string{"validated"},
|
Provides: []string{"validated"},
|
||||||
},
|
},
|
||||||
|
"fake/llm-validator": {
|
||||||
|
Key: "fake/llm-validator",
|
||||||
|
Stage: pipeline.StageValidate,
|
||||||
|
Requires: []string{"normalized"},
|
||||||
|
Provides: []string{"validated"},
|
||||||
|
},
|
||||||
"json": {
|
"json": {
|
||||||
Key: "json",
|
Key: "json",
|
||||||
Stage: pipeline.StageOutput,
|
Stage: pipeline.StageOutput,
|
||||||
@@ -426,6 +501,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
mustRegisterMerger(t, mergers, specs["appendorder"])
|
mustRegisterMerger(t, mergers, specs["appendorder"])
|
||||||
mustRegisterNormalizer(t, normalizers, specs["noop"])
|
mustRegisterNormalizer(t, normalizers, specs["noop"])
|
||||||
mustRegisterValidator(t, validators, specs["fake/validator"])
|
mustRegisterValidator(t, validators, specs["fake/validator"])
|
||||||
|
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
|
||||||
mustRegisterOutput(t, outputs, specs["json"])
|
mustRegisterOutput(t, outputs, specs["json"])
|
||||||
|
|
||||||
return pipeline.ModuleCatalog{
|
return pipeline.ModuleCatalog{
|
||||||
@@ -435,6 +511,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
Validators: validators,
|
Validators: validators,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -476,7 +553,12 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
|
|||||||
|
|
||||||
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
executionClass := contracts.ExecutionClassDeterministic
|
||||||
|
if spec.Key == "fake/llm-validator" {
|
||||||
|
executionClass = contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
|
||||||
|
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
||||||
t.Fatalf("register validator: %v", err)
|
t.Fatalf("register validator: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -255,20 +255,15 @@ func (validator compositionValidator) Name() string {
|
|||||||
return "generic-validator"
|
return "generic-validator"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
return contracts.ExecutionClassDeterministic
|
||||||
for _, candidate := range req.Candidates {
|
|
||||||
decisions = append(decisions, contracts.ValidationDecision{
|
|
||||||
CandidateIndex: candidate.Index,
|
|
||||||
Approved: true,
|
|
||||||
ReasonCode: "accepted",
|
|
||||||
Message: "candidate accepted",
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
return contracts.ValidationResult{
|
return contracts.ValidationResult{
|
||||||
ValidatorName: validator.Name(),
|
Approved: true,
|
||||||
Decisions: decisions,
|
ReasonCode: "accepted",
|
||||||
|
Message: "output accepted",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -203,20 +203,37 @@ type RawPayload struct {
|
|||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RawValidationRequest struct {
|
type ExecutionClass string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ExecutionClassDeterministic ExecutionClass = "deterministic"
|
||||||
|
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ValidationRequest struct {
|
||||||
Stage string `json:"stage"`
|
Stage string `json:"stage"`
|
||||||
LaneID string `json:"lane_id,omitempty"`
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
SourceID string `json:"source_id,omitempty"`
|
SourceID string `json:"source_id,omitempty"`
|
||||||
ChunkID string `json:"chunk_id,omitempty"`
|
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
Schema ResponseSchema `json:"schema,omitempty"`
|
Schema ResponseSchema `json:"schema,omitempty"`
|
||||||
Payload RawPayload `json:"payload"`
|
Payload RawPayload `json:"payload"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
ChunkID string `json:"chunk_id,omitempty"`
|
||||||
|
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||||
|
Chunk *SourceChunk `json:"chunk,omitempty"`
|
||||||
|
Chunks []SourceChunk `json:"chunks,omitempty"`
|
||||||
|
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
|
||||||
|
MergeOutput MergeOutput `json:"merge_output,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RawValidationResult struct {
|
type ValidationResult struct {
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code,omitempty"`
|
ReasonCode string `json:"reason_code,omitempty"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
@@ -224,15 +241,17 @@ type RawValidationResult struct {
|
|||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RawValidator interface {
|
type Validator interface {
|
||||||
Name() string
|
Name() string
|
||||||
ValidateRaw(ctx context.Context, req RawValidationRequest) (RawValidationResult, error)
|
ExecutionClass() ExecutionClass
|
||||||
|
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResponseSchema struct {
|
type ResponseSchema struct {
|
||||||
ID string `json:"id,omitempty"`
|
ID string `json:"id,omitempty"`
|
||||||
Name string `json:"name,omitempty"`
|
Name string `json:"name,omitempty"`
|
||||||
Version string `json:"version,omitempty"`
|
Version string `json:"version,omitempty"`
|
||||||
|
JSONSchema []byte `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ExtractOutput struct {
|
type ExtractOutput struct {
|
||||||
@@ -308,33 +327,6 @@ type Normalizer interface {
|
|||||||
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ValidationRequest struct {
|
|
||||||
Source *source.SourceDocument `json:"-"`
|
|
||||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
|
||||||
Options map[string]any `json:"options,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ValidationDecision struct {
|
|
||||||
CandidateIndex int `json:"candidate_index"`
|
|
||||||
Approved bool `json:"approved"`
|
|
||||||
ReasonCode string `json:"reason_code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ValidationResult struct {
|
|
||||||
ValidatorName string `json:"validator_name"`
|
|
||||||
Decisions []ValidationDecision `json:"decisions"`
|
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Validator interface {
|
|
||||||
Name() string
|
|
||||||
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Warning struct {
|
type Warning struct {
|
||||||
Scope string `json:"scope,omitempty"`
|
Scope string `json:"scope,omitempty"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
|
|||||||
@@ -322,6 +322,32 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
|
||||||
|
schema := ResponseSchema{
|
||||||
|
ID: "schema-id",
|
||||||
|
Name: "schema-name",
|
||||||
|
Version: "v1",
|
||||||
|
JSONSchema: []byte(`{"type":"object"}`),
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(schema)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got["id"] != "schema-id" || got["name"] != "schema-name" || got["version"] != "v1" {
|
||||||
|
t.Fatalf("encoded schema = %#v, want schema provenance", got)
|
||||||
|
}
|
||||||
|
if _, ok := got["json_schema"]; ok {
|
||||||
|
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
|
||||||
|
}
|
||||||
|
if _, ok := got["JSONSchema"]; ok {
|
||||||
|
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||||
extractOutput := ExtractOutput{
|
extractOutput := ExtractOutput{
|
||||||
LaneID: "generic-lane",
|
LaneID: "generic-lane",
|
||||||
@@ -577,20 +603,15 @@ func (validator fakeValidator) Name() string {
|
|||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
|
func (validator fakeValidator) ExecutionClass() ExecutionClass {
|
||||||
decisions := make([]ValidationDecision, 0, len(req.Candidates))
|
return ExecutionClassDeterministic
|
||||||
for _, candidate := range req.Candidates {
|
|
||||||
decisions = append(decisions, ValidationDecision{
|
|
||||||
CandidateIndex: candidate.Index,
|
|
||||||
Approved: true,
|
|
||||||
ReasonCode: "accepted",
|
|
||||||
Message: "candidate accepted",
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
|
||||||
return ValidationResult{
|
return ValidationResult{
|
||||||
ValidatorName: validator.name,
|
Approved: true,
|
||||||
Decisions: decisions,
|
ReasonCode: "accepted",
|
||||||
|
Message: "output accepted",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -386,6 +386,10 @@ func (validator registryValidator) Name() string {
|
|||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
return contracts.ValidationResult{}, nil
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,35 @@ type ModuleBinding struct {
|
|||||||
Retries int `json:"retries,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
References map[string]string `json:"references,omitempty"`
|
References map[string]string `json:"references,omitempty"`
|
||||||
|
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorOverride struct {
|
||||||
|
Set bool `json:"set,omitempty"`
|
||||||
|
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||||
|
type moduleBindingJSON struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
Retries int `json:"retries,omitempty"`
|
||||||
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]string `json:"references,omitempty"`
|
||||||
|
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
out := moduleBindingJSON{
|
||||||
|
Module: binding.Module,
|
||||||
|
LLMProfile: binding.LLMProfile,
|
||||||
|
Retries: binding.Retries,
|
||||||
|
Options: binding.Options,
|
||||||
|
References: binding.References,
|
||||||
|
}
|
||||||
|
if binding.Validators.Set {
|
||||||
|
validators := cloneModuleBindings(binding.Validators.Validators)
|
||||||
|
out.Validators = &validators
|
||||||
|
}
|
||||||
|
return json.Marshal(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
@@ -83,6 +112,18 @@ type ResolvedArtifactLane struct {
|
|||||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResolvedValidatorChain struct {
|
||||||
|
Stage ModuleStage `json:"stage"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
Validators []ResolvedValidator `json:"validators"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ResolvedValidator struct {
|
||||||
|
Binding ModuleBinding `json:"binding"`
|
||||||
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type ResolvedPipeline struct {
|
type ResolvedPipeline struct {
|
||||||
ID string
|
ID string
|
||||||
Digest string
|
Digest string
|
||||||
@@ -90,6 +131,7 @@ type ResolvedPipeline struct {
|
|||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||||
ArtifactLanes []ResolvedArtifactLane
|
ArtifactLanes []ResolvedArtifactLane
|
||||||
|
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||||
Output ModuleBinding
|
Output ModuleBinding
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,6 +142,7 @@ type ModuleCatalog struct {
|
|||||||
Mergers *MergerRegistry
|
Mergers *MergerRegistry
|
||||||
Normalizers *NormalizerRegistry
|
Normalizers *NormalizerRegistry
|
||||||
Validators *ValidatorRegistry
|
Validators *ValidatorRegistry
|
||||||
|
ValidatorChains *ValidatorChainRegistry
|
||||||
Outputs *OutputEncoderRegistry
|
Outputs *OutputEncoderRegistry
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,15 +215,21 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||||
}
|
}
|
||||||
|
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
|
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
|
||||||
outputCapabilities := capabilities.clone()
|
outputCapabilities := capabilities.clone()
|
||||||
|
|
||||||
for _, laneID := range selectedLaneIDs {
|
for _, laneID := range selectedLaneIDs {
|
||||||
laneProfile := lanesByID[laneID]
|
laneProfile := lanesByID[laneID]
|
||||||
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
|
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
|
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
|
||||||
|
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
|
||||||
outputCapabilities.addSet(laneCapabilities)
|
outputCapabilities.addSet(laneCapabilities)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,7 +257,7 @@ func resolveArtifactLane(
|
|||||||
options ResolveOptions,
|
options ResolveOptions,
|
||||||
inherited capabilitySet,
|
inherited capabilitySet,
|
||||||
catalog ModuleCatalog,
|
catalog ModuleCatalog,
|
||||||
) (ResolvedArtifactLane, capabilitySet, error) {
|
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
|
||||||
lane := ResolvedArtifactLane{
|
lane := ResolvedArtifactLane{
|
||||||
ID: laneID,
|
ID: laneID,
|
||||||
Extract: resolveBinding(profile.Extract, ""),
|
Extract: resolveBinding(profile.Extract, ""),
|
||||||
@@ -217,17 +266,17 @@ func resolveArtifactLane(
|
|||||||
Validators: resolveBindings(profile.Validators, ""),
|
Validators: resolveBindings(profile.Validators, ""),
|
||||||
}
|
}
|
||||||
if lane.Extract.Module == "" {
|
if lane.Extract.Module == "" {
|
||||||
return ResolvedArtifactLane{}, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
capabilities := inherited.clone()
|
capabilities := inherited.clone()
|
||||||
|
|
||||||
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
|
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||||
}
|
}
|
||||||
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
||||||
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
@@ -241,17 +290,17 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||||
capabilities.add(extractSpec.Provides...)
|
capabilities.add(extractSpec.Provides...)
|
||||||
|
|
||||||
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
|
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
||||||
}
|
}
|
||||||
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
PipelineID: pipelineID,
|
PipelineID: pipelineID,
|
||||||
@@ -264,17 +313,17 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||||
capabilities.add(mergeSpec.Provides...)
|
capabilities.add(mergeSpec.Provides...)
|
||||||
|
|
||||||
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
|
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
||||||
}
|
}
|
||||||
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
PipelineID: pipelineID,
|
PipelineID: pipelineID,
|
||||||
@@ -287,20 +336,108 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||||
capabilities.add(normalizeSpec.Provides...)
|
capabilities.add(normalizeSpec.Provides...)
|
||||||
|
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return ResolvedArtifactLane{}, nil, configuredValidatorsError(pipelineID, laneID)
|
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return lane, capabilities, nil
|
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}
|
||||||
|
|
||||||
|
return lane, validatorChains, capabilities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func configuredValidatorsError(pipelineID string, laneID string) error {
|
func configuredValidatorsError(pipelineID string, laneID string) error {
|
||||||
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
|
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
|
||||||
|
chain := ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
ModuleKey: strings.TrimSpace(module),
|
||||||
|
}
|
||||||
|
if chain.ModuleKey == "" {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
|
||||||
|
}
|
||||||
|
switch stage {
|
||||||
|
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||||
|
default:
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
var bindings []ModuleBinding
|
||||||
|
if override.Set {
|
||||||
|
bindings = cloneModuleBindings(override.Validators)
|
||||||
|
} else if catalog.ValidatorChains != nil {
|
||||||
|
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
|
||||||
|
}
|
||||||
|
if len(bindings) == 0 {
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
if catalog.Validators == nil {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
|
||||||
|
}
|
||||||
|
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
|
||||||
|
for _, validator := range bindings {
|
||||||
|
spec, ok := catalog.Validators.Spec(validator.Module)
|
||||||
|
if !ok {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||||
|
}
|
||||||
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator),
|
||||||
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
|
||||||
|
if len(chains) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ResolvedValidatorChain, len(chains))
|
||||||
|
for i, chain := range chains {
|
||||||
|
out[i] = ResolvedValidatorChain{
|
||||||
|
Stage: chain.Stage,
|
||||||
|
LaneID: strings.TrimSpace(chain.LaneID),
|
||||||
|
ModuleKey: strings.TrimSpace(chain.ModuleKey),
|
||||||
|
Validators: cloneResolvedValidators(chain.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
|
||||||
|
if len(validators) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ResolvedValidator, len(validators))
|
||||||
|
for i, validator := range validators {
|
||||||
|
out[i] = ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator.Binding),
|
||||||
|
ExecutionClass: validator.ExecutionClass,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
||||||
@@ -625,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
|||||||
Retries: binding.Retries,
|
Retries: binding.Retries,
|
||||||
Options: cloneOptions(binding.Options),
|
Options: cloneOptions(binding.Options),
|
||||||
References: normalizeReferenceMap(binding.References),
|
References: normalizeReferenceMap(binding.References),
|
||||||
|
Validators: cloneValidatorOverride(binding.Validators),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -110,6 +110,215 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {Extract: Binding("event-extractor")},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resolved.ValidatorChains) != 4 {
|
||||||
|
t.Fatalf("len(ValidatorChains) = %d, want chunk plus lane extract/merge/normalize", len(resolved.ValidatorChains))
|
||||||
|
}
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 1 {
|
||||||
|
t.Fatalf("extract validators = %#v, want one validator", extractChain.Validators)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].Binding.Module != "grounded" {
|
||||||
|
t.Fatalf("extract validator key = %q, want grounded", extractChain.Validators[0].Binding.Module)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("extract validator execution class = %q, want deterministic", extractChain.Validators[0].ExecutionClass)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", DefaultChunkModule)
|
||||||
|
if chunkChain == nil {
|
||||||
|
t.Fatal("chunk validator chain not found")
|
||||||
|
}
|
||||||
|
if len(chunkChain.Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validators = %#v, want explicit empty chain", chunkChain.Validators)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Module: DefaultNormalizeModule,
|
||||||
|
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "invalid-chain",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {Extract: Binding("event-extractor")},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing-validator") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profile := PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{
|
||||||
|
{Module: "second-validator", LLMProfile: "careful"},
|
||||||
|
Binding("grounded"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 2 {
|
||||||
|
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
|
||||||
|
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
|
||||||
|
}
|
||||||
|
if extractChain.Validators[1].Binding.Module != "grounded" {
|
||||||
|
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profile := PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{Set: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 0 {
|
||||||
|
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing-validator") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{
|
||||||
|
{Module: "grounded", LLMProfile: "careful"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
||||||
profile := multiLaneProfile()
|
profile := multiLaneProfile()
|
||||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
||||||
@@ -778,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
}
|
}
|
||||||
assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported")
|
assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
|
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
|
||||||
@@ -955,6 +1164,15 @@ func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName str
|
|||||||
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func findResolvedValidatorChain(chains []ResolvedValidatorChain, stage ModuleStage, laneID string, module string) *ResolvedValidatorChain {
|
||||||
|
for i := range chains {
|
||||||
|
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||||
|
return &chains[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1000,6 +1218,7 @@ func emptyProfileCatalog() ModuleCatalog {
|
|||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
Validators: NewValidatorRegistry(),
|
Validators: NewValidatorRegistry(),
|
||||||
|
ValidatorChains: NewValidatorChainRegistry(),
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1043,7 +1262,8 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
|||||||
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
||||||
}
|
}
|
||||||
case StageValidate:
|
case StageValidate:
|
||||||
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||||
|
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||||
}
|
}
|
||||||
case StageOutput:
|
case StageOutput:
|
||||||
@@ -1056,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||||
|
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func profileInputConstructor(key string) InputAdapterConstructor {
|
func profileInputConstructor(key string) InputAdapterConstructor {
|
||||||
return func() (contracts.InputAdapter, error) {
|
return func() (contracts.InputAdapter, error) {
|
||||||
return profileInputAdapter{key: key}, nil
|
return profileInputAdapter{key: key}, nil
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type rawValidationKey struct {
|
|
||||||
stage ModuleStage
|
|
||||||
module string
|
|
||||||
}
|
|
||||||
|
|
||||||
type RawValidationRegistry struct {
|
|
||||||
chains map[rawValidationKey][]contracts.RawValidator
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRawValidationRegistry() *RawValidationRegistry {
|
|
||||||
return &RawValidationRegistry{
|
|
||||||
chains: make(map[rawValidationKey][]contracts.RawValidator),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.RawValidator) error {
|
|
||||||
if r == nil {
|
|
||||||
return fmt.Errorf("raw validation registry must not be nil")
|
|
||||||
}
|
|
||||||
normalizedModule := strings.TrimSpace(module)
|
|
||||||
if normalizedModule == "" {
|
|
||||||
return fmt.Errorf("raw validation module key must not be empty")
|
|
||||||
}
|
|
||||||
switch stage {
|
|
||||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("raw validation stage %q is not supported", stage)
|
|
||||||
}
|
|
||||||
if len(validators) == 0 {
|
|
||||||
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
|
|
||||||
}
|
|
||||||
|
|
||||||
chain := make([]contracts.RawValidator, 0, len(validators))
|
|
||||||
for i, validator := range validators {
|
|
||||||
if validator == nil {
|
|
||||||
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(validator.Name()) == "" {
|
|
||||||
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
|
|
||||||
}
|
|
||||||
chain = append(chain, validator)
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.chains == nil {
|
|
||||||
r.chains = make(map[rawValidationKey][]contracts.RawValidator)
|
|
||||||
}
|
|
||||||
key := rawValidationKey{stage: stage, module: normalizedModule}
|
|
||||||
if _, exists := r.chains[key]; exists {
|
|
||||||
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
|
|
||||||
}
|
|
||||||
r.chains[key] = append([]contracts.RawValidator(nil), chain...)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.RawValidator {
|
|
||||||
if r == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
chain := r.chains[rawValidationKey{stage: stage, module: strings.TrimSpace(module)}]
|
|
||||||
if len(chain) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return append([]contracts.RawValidator(nil), chain...)
|
|
||||||
}
|
|
||||||
@@ -24,7 +24,7 @@ type Registries struct {
|
|||||||
Mergers *MergerRegistry
|
Mergers *MergerRegistry
|
||||||
Normalizers *NormalizerRegistry
|
Normalizers *NormalizerRegistry
|
||||||
Validators *ValidatorRegistry
|
Validators *ValidatorRegistry
|
||||||
RawValidators *RawValidationRegistry
|
ValidatorChains *ValidatorChainRegistry
|
||||||
Outputs *OutputEncoderRegistry
|
Outputs *OutputEncoderRegistry
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,7 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||||
}
|
}
|
||||||
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, attempt)
|
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt)
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
return false, rejection, err
|
return false, rejection, err
|
||||||
}
|
}
|
||||||
@@ -237,9 +237,15 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
sourceID: doc.ID,
|
sourceID: doc.ID,
|
||||||
chunkID: chunk.ID,
|
chunkID: chunk.ID,
|
||||||
chunkIndex: chunk.Index,
|
chunkIndex: chunk.Index,
|
||||||
|
chunk: &chunk,
|
||||||
|
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||||
|
sessionID: sessionID,
|
||||||
|
references: lane.ExtractReferences.ReferenceSet,
|
||||||
|
llmClient: input.LLMClient,
|
||||||
schema: extractOutput.Schema,
|
schema: extractOutput.Schema,
|
||||||
payload: extractOutput.Payload,
|
payload: extractOutput.Payload,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -293,9 +299,15 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
moduleKey: merger.Key(),
|
moduleKey: merger.Key(),
|
||||||
source: doc,
|
source: doc,
|
||||||
sourceID: doc.ID,
|
sourceID: doc.ID,
|
||||||
|
sourceInput: sourceInput.Clone(),
|
||||||
|
sessionID: sessionID,
|
||||||
|
references: lane.MergeReferences.ReferenceSet,
|
||||||
|
llmClient: input.LLMClient,
|
||||||
schema: mergeOutput.Schema,
|
schema: mergeOutput.Schema,
|
||||||
payload: mergeOutput.Payload,
|
payload: mergeOutput.Payload,
|
||||||
|
extractOutputs: extractOutputs,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -343,9 +355,15 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
moduleKey: normalizer.Key(),
|
moduleKey: normalizer.Key(),
|
||||||
source: doc,
|
source: doc,
|
||||||
sourceID: doc.ID,
|
sourceID: doc.ID,
|
||||||
|
sourceInput: sourceInput.Clone(),
|
||||||
|
sessionID: sessionID,
|
||||||
|
references: lane.NormalizeReferences.ReferenceSet,
|
||||||
|
llmClient: input.LLMClient,
|
||||||
schema: normalizeOutput.Schema,
|
schema: normalizeOutput.Schema,
|
||||||
payload: normalizeOutput.Payload,
|
payload: normalizeOutput.Payload,
|
||||||
|
mergeOutput: acceptedMerge,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -373,11 +391,20 @@ type rawValidationTarget struct {
|
|||||||
moduleKey string
|
moduleKey string
|
||||||
source *source.SourceDocument
|
source *source.SourceDocument
|
||||||
sourceID string
|
sourceID string
|
||||||
|
sourceInput contracts.LLMInputMaterial
|
||||||
|
sessionID string
|
||||||
|
references contracts.ReferenceSet
|
||||||
|
llmClient contracts.StructuredLLMClient
|
||||||
chunkID string
|
chunkID string
|
||||||
chunkIndex int
|
chunkIndex int
|
||||||
|
chunk *contracts.SourceChunk
|
||||||
|
chunks []contracts.SourceChunk
|
||||||
schema contracts.ResponseSchema
|
schema contracts.ResponseSchema
|
||||||
payload contracts.RawPayload
|
payload contracts.RawPayload
|
||||||
|
extractOutputs []contracts.ExtractOutput
|
||||||
|
mergeOutput contracts.MergeOutput
|
||||||
metadata map[string]any
|
metadata map[string]any
|
||||||
|
chains []ResolvedValidatorChain
|
||||||
attempt int
|
attempt int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,21 +455,29 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
|
|||||||
return false, lastRejection, nil
|
return false, lastRejection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, attempt int) (*contracts.RejectedOutput, error) {
|
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) (*contracts.RejectedOutput, error) {
|
||||||
for _, chunk := range chunks {
|
for index := range chunks {
|
||||||
|
chunk := chunks[index]
|
||||||
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||||
stage: StageChunk,
|
stage: StageChunk,
|
||||||
moduleKey: moduleKey,
|
moduleKey: moduleKey,
|
||||||
source: doc,
|
source: doc,
|
||||||
sourceID: doc.ID,
|
sourceID: doc.ID,
|
||||||
|
sourceInput: sourceInput.Clone(),
|
||||||
|
sessionID: sessionID,
|
||||||
|
references: references,
|
||||||
|
llmClient: llmClient,
|
||||||
chunkID: chunk.ID,
|
chunkID: chunk.ID,
|
||||||
chunkIndex: chunk.Index,
|
chunkIndex: chunk.Index,
|
||||||
|
chunk: &chunk,
|
||||||
|
chunks: chunks,
|
||||||
payload: contracts.RawPayload{
|
payload: contracts.RawPayload{
|
||||||
Content: append([]byte(nil), chunk.Content...),
|
Content: append([]byte(nil), chunk.Content...),
|
||||||
MediaType: chunk.MediaType,
|
MediaType: chunk.MediaType,
|
||||||
Metadata: cloneMetadata(chunk.Metadata),
|
Metadata: cloneMetadata(chunk.Metadata),
|
||||||
},
|
},
|
||||||
metadata: metadata,
|
metadata: metadata,
|
||||||
|
chains: chains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -456,27 +491,22 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||||
validators := r.registries.RawValidators.Validators(target.stage, target.moduleKey)
|
chain := resolvedValidatorChain(target.stage, target.laneID, target.moduleKey, target.chains)
|
||||||
if len(validators) == 0 {
|
if len(chain.Validators) == 0 {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
if r.registries.Validators == nil {
|
||||||
request := contracts.RawValidationRequest{
|
return nil, nil, fmt.Errorf("validator registry must not be nil")
|
||||||
Stage: string(target.stage),
|
|
||||||
LaneID: target.laneID,
|
|
||||||
ModuleKey: target.moduleKey,
|
|
||||||
Source: target.source,
|
|
||||||
SourceID: target.sourceID,
|
|
||||||
ChunkID: target.chunkID,
|
|
||||||
ChunkIndex: target.chunkIndex,
|
|
||||||
Schema: target.schema,
|
|
||||||
Payload: cloneRawPayload(target.payload),
|
|
||||||
Metadata: cloneMetadata(target.metadata),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var warnings []contracts.Warning
|
var warnings []contracts.Warning
|
||||||
for _, validator := range validators {
|
for _, validatorBinding := range chain.Validators {
|
||||||
result, err := validator.ValidateRaw(ctx, request)
|
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
|
||||||
|
}
|
||||||
|
request := target.validationRequest(validatorBinding.Binding)
|
||||||
|
result, err := validator.Validate(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
||||||
}
|
}
|
||||||
@@ -507,6 +537,56 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
|
|||||||
return warnings, nil, nil
|
return warnings, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (target rawValidationTarget) validationRequest(binding ModuleBinding) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Stage: string(target.stage),
|
||||||
|
LaneID: target.laneID,
|
||||||
|
ModuleKey: target.moduleKey,
|
||||||
|
Source: target.source,
|
||||||
|
SourceID: target.sourceID,
|
||||||
|
SourceInput: target.sourceInput.Clone(),
|
||||||
|
SessionID: target.sessionID,
|
||||||
|
References: CloneReferenceSet(target.references),
|
||||||
|
LLMClient: target.llmClient,
|
||||||
|
LLMProfile: binding.LLMProfile,
|
||||||
|
Options: cloneOptions(binding.Options),
|
||||||
|
Metadata: cloneMetadata(target.metadata),
|
||||||
|
Schema: cloneResponseSchema(target.schema),
|
||||||
|
Payload: cloneRawPayload(target.payload),
|
||||||
|
ChunkID: target.chunkID,
|
||||||
|
ChunkIndex: target.chunkIndex,
|
||||||
|
Chunk: cloneSourceChunkPtr(target.chunk),
|
||||||
|
Chunks: cloneSourceChunks(target.chunks),
|
||||||
|
ExtractOutputs: cloneExtractOutputs(target.extractOutputs),
|
||||||
|
MergeOutput: cloneMergeOutput(target.mergeOutput),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||||
|
for _, chain := range chains {
|
||||||
|
if chain.Stage != stage {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if chain.ModuleKey != moduleKey {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return ResolvedValidatorChain{
|
||||||
|
Stage: chain.Stage,
|
||||||
|
LaneID: chain.LaneID,
|
||||||
|
ModuleKey: chain.ModuleKey,
|
||||||
|
Validators: cloneResolvedValidators(chain.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
ModuleKey: strings.TrimSpace(moduleKey),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
|
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
|
||||||
if r.registries.Inputs == nil {
|
if r.registries.Inputs == nil {
|
||||||
return fmt.Errorf("input registry must not be nil")
|
return fmt.Errorf("input registry must not be nil")
|
||||||
@@ -562,7 +642,7 @@ func validateRunInput(input RunInput) error {
|
|||||||
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
|
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
|
||||||
}
|
}
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID)
|
return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -586,6 +666,7 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
Chunker: pipeline.Chunk.Module,
|
Chunker: pipeline.Chunk.Module,
|
||||||
OutputEncoder: pipeline.Output.Module,
|
OutputEncoder: pipeline.Output.Module,
|
||||||
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
||||||
|
ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
StartedAt: timePtr(startedAt),
|
StartedAt: timePtr(startedAt),
|
||||||
References: ReferenceProvenance(pipeline),
|
References: ReferenceProvenance(pipeline),
|
||||||
@@ -607,6 +688,29 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
return manifest
|
return manifest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.ValidatorChainManifest {
|
||||||
|
if len(chains) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
manifests := make([]artifacts.ValidatorChainManifest, 0, len(chains))
|
||||||
|
for _, chain := range chains {
|
||||||
|
manifest := artifacts.ValidatorChainManifest{
|
||||||
|
Stage: string(chain.Stage),
|
||||||
|
LaneID: chain.LaneID,
|
||||||
|
ModuleKey: chain.ModuleKey,
|
||||||
|
Validators: make([]artifacts.ValidatorManifest, 0, len(chain.Validators)),
|
||||||
|
}
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
manifest.Validators = append(manifest.Validators, artifacts.ValidatorManifest{
|
||||||
|
Key: validator.Binding.Module,
|
||||||
|
ExecutionClass: string(validator.ExecutionClass),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
manifests = append(manifests, manifest)
|
||||||
|
}
|
||||||
|
return manifests
|
||||||
|
}
|
||||||
|
|
||||||
func failOutput(output RunOutput) RunOutput {
|
func failOutput(output RunOutput) RunOutput {
|
||||||
if output.Manifest.PipelineID != "" {
|
if output.Manifest.PipelineID != "" {
|
||||||
populateRawOutputManifest(&output)
|
populateRawOutputManifest(&output)
|
||||||
@@ -908,7 +1012,50 @@ func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||||
|
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
|
||||||
|
return schema
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceChunkPtr(chunk *contracts.SourceChunk) *contracts.SourceChunk {
|
||||||
|
if chunk == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := cloneSourceChunk(*chunk)
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceChunk(chunk contracts.SourceChunk) contracts.SourceChunk {
|
||||||
|
chunk.Content = append([]byte(nil), chunk.Content...)
|
||||||
|
chunk.Units = cloneSourceUnits(chunk.Units)
|
||||||
|
chunk.Metadata = cloneMetadata(chunk.Metadata)
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceChunks(chunks []contracts.SourceChunk) []contracts.SourceChunk {
|
||||||
|
if len(chunks) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]contracts.SourceChunk, 0, len(chunks))
|
||||||
|
for _, chunk := range chunks {
|
||||||
|
out = append(out, cloneSourceChunk(chunk))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||||
|
if len(units) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]source.SourceUnit, 0, len(units))
|
||||||
|
for _, unit := range units {
|
||||||
|
out = append(out, cloneSourceUnit(unit))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
||||||
|
output.Schema = cloneResponseSchema(output.Schema)
|
||||||
output.Payload = cloneRawPayload(output.Payload)
|
output.Payload = cloneRawPayload(output.Payload)
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
@@ -925,11 +1072,13 @@ func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractO
|
|||||||
}
|
}
|
||||||
|
|
||||||
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
|
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
|
||||||
|
output.Schema = cloneResponseSchema(output.Schema)
|
||||||
output.Payload = cloneRawPayload(output.Payload)
|
output.Payload = cloneRawPayload(output.Payload)
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
|
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
|
||||||
|
output.Schema = cloneResponseSchema(output.Schema)
|
||||||
output.Payload = cloneRawPayload(output.Payload)
|
output.Payload = cloneRawPayload(output.Payload)
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -763,6 +763,110 @@ func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPassesValidationRequestContextToValidators(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
chunkValidator := &runnerChainValidator{name: "chain-chunk"}
|
||||||
|
extractValidator := &runnerChainValidator{name: "chain-extract", executionClass: contracts.ExecutionClassLLMBacked}
|
||||||
|
mergeValidator := &runnerChainValidator{name: "chain-merge"}
|
||||||
|
normalizeValidator := &runnerChainValidator{name: "chain-normalize"}
|
||||||
|
modules.validators[chunkValidator.name] = chunkValidator
|
||||||
|
modules.validators[extractValidator.name] = extractValidator
|
||||||
|
modules.validators[mergeValidator.name] = mergeValidator
|
||||||
|
modules.validators[normalizeValidator.name] = normalizeValidator
|
||||||
|
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
|
||||||
|
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "extract reference text")
|
||||||
|
pipeline.ArtifactLanes[0].MergeReferences.ReferenceSet = testReferenceSet("merge_notes", "merge reference text")
|
||||||
|
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageChunk, "", "chunk", resolvedValidatorForTest(chunkValidator))
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", ResolvedValidator{
|
||||||
|
Binding: ModuleBinding{Module: extractValidator.name, LLMProfile: "validator-profile", Options: map[string]any{"strict": true}},
|
||||||
|
ExecutionClass: extractValidator.ExecutionClass(),
|
||||||
|
})
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageMerge, "alpha", "merge", resolvedValidatorForTest(mergeValidator))
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageNormalize, "alpha", "normalize", resolvedValidatorForTest(normalizeValidator))
|
||||||
|
|
||||||
|
rawInput := []byte("{\"source\":\"exact bytes\"}")
|
||||||
|
llmClient := fakeLLMClient{}
|
||||||
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
|
||||||
|
Pipeline: pipeline,
|
||||||
|
Path: "session.json",
|
||||||
|
RawInput: rawInput,
|
||||||
|
LLMClient: llmClient,
|
||||||
|
SessionID: "session-123",
|
||||||
|
Metadata: map[string]any{"request": "test"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(chunkValidator.requests) != 2 {
|
||||||
|
t.Fatalf("chunk validator requests = %d, want one per chunk", len(chunkValidator.requests))
|
||||||
|
}
|
||||||
|
chunkReq := chunkValidator.requests[0]
|
||||||
|
if chunkReq.Stage != string(StageChunk) || chunkReq.ModuleKey != "chunk" || chunkReq.SourceID != "source-1" || chunkReq.SessionID != "session-123" {
|
||||||
|
t.Fatalf("chunk validation request = %#v, want stage/module/source/session provenance", chunkReq)
|
||||||
|
}
|
||||||
|
if chunkReq.LLMClient == nil || string(chunkReq.SourceInput.Content) != string(rawInput) {
|
||||||
|
t.Fatalf("chunk validation source/client = %#v, want full source input and LLM client", chunkReq.SourceInput)
|
||||||
|
}
|
||||||
|
if chunkReq.Chunk == nil || chunkReq.Chunk.ID != "chunk-0" || len(chunkReq.Chunks) != 2 || string(chunkReq.Payload.Content) != string(chunkReq.Chunk.Content) {
|
||||||
|
t.Fatalf("chunk validation chunk fields = %#v chunks=%#v payload=%s, want chunk payload and all chunks", chunkReq.Chunk, chunkReq.Chunks, chunkReq.Payload.Content)
|
||||||
|
}
|
||||||
|
if item := chunkReq.References.Slots["scene_guide"].Items[0]; string(item.Content) != "chunk reference text" {
|
||||||
|
t.Fatalf("chunk validation references = %#v, want chunk references", chunkReq.References)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(extractValidator.requests) != 2 {
|
||||||
|
t.Fatalf("extract validator requests = %d, want one per chunk", len(extractValidator.requests))
|
||||||
|
}
|
||||||
|
extractReq := extractValidator.requests[0]
|
||||||
|
if extractReq.Stage != string(StageExtract) || extractReq.LaneID != "alpha" || extractReq.ModuleKey != "extract-alpha" || extractReq.ChunkID != "chunk-0" || extractReq.ChunkIndex != 0 {
|
||||||
|
t.Fatalf("extract validation request = %#v, want extract provenance", extractReq)
|
||||||
|
}
|
||||||
|
if extractReq.LLMProfile != "validator-profile" || extractReq.Options["strict"] != true || extractReq.Metadata["request"] != "test" {
|
||||||
|
t.Fatalf("extract validator binding fields = profile %q options %#v metadata %#v", extractReq.LLMProfile, extractReq.Options, extractReq.Metadata)
|
||||||
|
}
|
||||||
|
if extractReq.Chunk == nil || string(extractReq.SourceInput.Content) != string(extractReq.Chunk.Content) {
|
||||||
|
t.Fatalf("extract source input = %#v chunk=%#v, want chunk material", extractReq.SourceInput, extractReq.Chunk)
|
||||||
|
}
|
||||||
|
if item := extractReq.References.Slots["roster"].Items[0]; string(item.Content) != "extract reference text" {
|
||||||
|
t.Fatalf("extract validation references = %#v, want extract references", extractReq.References)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(mergeValidator.requests) != 1 {
|
||||||
|
t.Fatalf("merge validator requests = %d, want one", len(mergeValidator.requests))
|
||||||
|
}
|
||||||
|
mergeReq := mergeValidator.requests[0]
|
||||||
|
if mergeReq.Stage != string(StageMerge) || mergeReq.LaneID != "alpha" || len(mergeReq.ExtractOutputs) != 2 {
|
||||||
|
t.Fatalf("merge validation request = %#v, want lane and extract outputs", mergeReq)
|
||||||
|
}
|
||||||
|
if mergeReq.ExtractOutputs[0].ChunkID != "chunk-0" || string(mergeReq.SourceInput.Content) != string(rawInput) {
|
||||||
|
t.Fatalf("merge validation upstream/source = %#v source=%#v, want ordered extracts and source input", mergeReq.ExtractOutputs, mergeReq.SourceInput)
|
||||||
|
}
|
||||||
|
if item := mergeReq.References.Slots["merge_notes"].Items[0]; string(item.Content) != "merge reference text" {
|
||||||
|
t.Fatalf("merge validation references = %#v, want merge references", mergeReq.References)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(normalizeValidator.requests) != 1 {
|
||||||
|
t.Fatalf("normalize validator requests = %d, want one", len(normalizeValidator.requests))
|
||||||
|
}
|
||||||
|
normalizeReq := normalizeValidator.requests[0]
|
||||||
|
if normalizeReq.Stage != string(StageNormalize) || normalizeReq.LaneID != "alpha" || string(normalizeReq.MergeOutput.Payload.Content) != `{"merged":true}` {
|
||||||
|
t.Fatalf("normalize validation request = %#v, want merge output context", normalizeReq)
|
||||||
|
}
|
||||||
|
if item := normalizeReq.References.Slots["normalization_notes"].Items[0]; string(item.Content) != "normalize reference text" {
|
||||||
|
t.Fatalf("normalize validation references = %#v, want normalize references", normalizeReq.References)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkReq.Payload.Content[0] = 'X'
|
||||||
|
chunkReq.Chunks[0].Content[0] = 'Y'
|
||||||
|
if got := string(modules.chunker.chunks[0].Content); got != `{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}` {
|
||||||
|
t.Fatalf("validator request mutated original chunk content: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
|
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
|
||||||
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -887,10 +991,12 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -909,10 +1015,12 @@ func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -933,10 +1041,12 @@ func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
|
validator := &runnerChainValidator{name: "chain-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageMerge, "merge", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageMerge, "alpha", "merge", resolvedValidatorForTest(validator))
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -954,10 +1064,12 @@ func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
|
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
|
validator := &runnerChainValidator{name: "chain-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageNormalize, "normalize", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageNormalize, "alpha", "normalize", resolvedValidatorForTest(validator))
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -999,9 +1111,10 @@ func TestRunRetriesSameModuleInputAfterFrameworkError(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
pipeline := resolvedPipeline()
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
|
||||||
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
@@ -1023,9 +1136,10 @@ func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
|
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
pipeline := resolvedPipeline()
|
pipeline := resolvedPipeline()
|
||||||
|
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
|
||||||
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
@@ -1075,7 +1189,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) {
|
|||||||
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
|
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
|
||||||
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
|
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
|
||||||
})
|
})
|
||||||
assertRunError(t, err, "configured validators")
|
assertRunError(t, err, "extract.validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCollectsStageWarnings(t *testing.T) {
|
func TestRunCollectsStageWarnings(t *testing.T) {
|
||||||
@@ -1312,8 +1426,11 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
|||||||
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
|
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
|
||||||
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
|
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
|
||||||
}
|
}
|
||||||
if len(lane.Validators) != 0 {
|
if len(manifest.ValidatorChains) != 4 {
|
||||||
t.Fatalf("lane validators = %#v, want none", lane.Validators)
|
t.Fatalf("ValidatorChains = %#v, want four validation points", manifest.ValidatorChains)
|
||||||
|
}
|
||||||
|
if manifest.ValidatorChains[0].Stage != string(StageChunk) || manifest.ValidatorChains[0].ModuleKey != "chunk" || len(manifest.ValidatorChains[0].Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validator chain = %#v, want explicit empty chunk chain", manifest.ValidatorChains[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1454,6 +1571,12 @@ func resolvedPipeline() ResolvedPipeline {
|
|||||||
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
|
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
ValidatorChains: []ResolvedValidatorChain{
|
||||||
|
{Stage: StageChunk, ModuleKey: "chunk"},
|
||||||
|
{Stage: StageExtract, LaneID: "alpha", ModuleKey: "extract-alpha"},
|
||||||
|
{Stage: StageMerge, LaneID: "alpha", ModuleKey: "merge"},
|
||||||
|
{Stage: StageNormalize, LaneID: "alpha", ModuleKey: "normalize"},
|
||||||
|
},
|
||||||
Output: Binding("output"),
|
Output: Binding("output"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1493,8 +1616,7 @@ type runnerModules struct {
|
|||||||
extractors map[string]*runnerExtractor
|
extractors map[string]*runnerExtractor
|
||||||
mergers map[string]*runnerMerger
|
mergers map[string]*runnerMerger
|
||||||
normalizers map[string]*runnerNormalizer
|
normalizers map[string]*runnerNormalizer
|
||||||
validators map[string]*runnerValidator
|
validators map[string]contracts.Validator
|
||||||
rawValidators *RawValidationRegistry
|
|
||||||
output *runnerOutputEncoder
|
output *runnerOutputEncoder
|
||||||
inputBuildErr error
|
inputBuildErr error
|
||||||
chunkerBuildErr error
|
chunkerBuildErr error
|
||||||
@@ -1513,9 +1635,9 @@ func defaultRunnerModules() *runnerModules {
|
|||||||
normalizers: map[string]*runnerNormalizer{
|
normalizers: map[string]*runnerNormalizer{
|
||||||
"normalize": {key: "normalize"},
|
"normalize": {key: "normalize"},
|
||||||
},
|
},
|
||||||
validators: map[string]*runnerValidator{
|
validators: map[string]contracts.Validator{
|
||||||
"configured": {name: "configured"},
|
"configured": &runnerValidator{name: "configured"},
|
||||||
"second-validator": {name: "second-validator"},
|
"second-validator": &runnerValidator{name: "second-validator"},
|
||||||
},
|
},
|
||||||
output: &runnerOutputEncoder{
|
output: &runnerOutputEncoder{
|
||||||
key: "output",
|
key: "output",
|
||||||
@@ -1539,7 +1661,7 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
|
|||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
Validators: NewValidatorRegistry(),
|
Validators: NewValidatorRegistry(),
|
||||||
RawValidators: modules.rawValidators,
|
ValidatorChains: NewValidatorChainRegistry(),
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
|
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
|
||||||
@@ -1578,7 +1700,8 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
|
|||||||
}
|
}
|
||||||
for key, validator := range modules.validators {
|
for key, validator := range modules.validators {
|
||||||
validator := validator
|
validator := validator
|
||||||
if err := registries.Validators.Register(key, func() (contracts.Validator, error) { return validator, nil }); err != nil {
|
spec := ValidatorSpec{Key: key, ExecutionClass: validator.ExecutionClass()}
|
||||||
|
if err := registries.Validators.RegisterWithSpec(spec, func() (contracts.Validator, error) { return validator, nil }); err != nil {
|
||||||
t.Fatalf("register validator %q: %v", key, err)
|
t.Fatalf("register validator %q: %v", key, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1797,8 +1920,10 @@ func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts
|
|||||||
|
|
||||||
type runnerValidator struct {
|
type runnerValidator struct {
|
||||||
name string
|
name string
|
||||||
resultName string
|
executionClass contracts.ExecutionClass
|
||||||
decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision
|
approved []bool
|
||||||
|
reason string
|
||||||
|
message string
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
order *[]string
|
order *[]string
|
||||||
@@ -1806,26 +1931,34 @@ type runnerValidator struct {
|
|||||||
requests []contracts.ValidationRequest
|
requests []contracts.ValidationRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
type runnerRawValidator struct {
|
type runnerChainValidator struct {
|
||||||
name string
|
name string
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
approved []bool
|
approved []bool
|
||||||
reason string
|
reason string
|
||||||
message string
|
message string
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
calls int
|
calls int
|
||||||
requests []contracts.RawValidationRequest
|
requests []contracts.ValidationRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator *runnerRawValidator) Name() string {
|
func (validator *runnerChainValidator) Name() string {
|
||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contracts.RawValidationRequest) (contracts.RawValidationResult, error) {
|
func (validator *runnerChainValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
if validator.executionClass != "" {
|
||||||
|
return validator.executionClass
|
||||||
|
}
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator *runnerChainValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
validator.calls++
|
validator.calls++
|
||||||
validator.requests = append(validator.requests, req)
|
validator.requests = append(validator.requests, req)
|
||||||
if validator.err != nil {
|
if validator.err != nil {
|
||||||
return contracts.RawValidationResult{}, validator.err
|
return contracts.ValidationResult{}, validator.err
|
||||||
}
|
}
|
||||||
approved := true
|
approved := true
|
||||||
if len(validator.approved) > 0 {
|
if len(validator.approved) > 0 {
|
||||||
@@ -1835,7 +1968,7 @@ func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contra
|
|||||||
}
|
}
|
||||||
approved = validator.approved[index]
|
approved = validator.approved[index]
|
||||||
}
|
}
|
||||||
return contracts.RawValidationResult{
|
return contracts.ValidationResult{
|
||||||
Approved: approved,
|
Approved: approved,
|
||||||
ReasonCode: validator.reason,
|
ReasonCode: validator.reason,
|
||||||
Message: validator.message,
|
Message: validator.message,
|
||||||
@@ -1847,23 +1980,31 @@ func (validator *runnerValidator) Name() string {
|
|||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (validator *runnerValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
if validator.executionClass != "" {
|
||||||
|
return validator.executionClass
|
||||||
|
}
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
validator.calls++
|
validator.calls++
|
||||||
validator.requests = append(validator.requests, req)
|
validator.requests = append(validator.requests, req)
|
||||||
if validator.order != nil {
|
if validator.order != nil {
|
||||||
*validator.order = append(*validator.order, validator.name)
|
*validator.order = append(*validator.order, validator.name)
|
||||||
}
|
}
|
||||||
resultName := validator.resultName
|
approved := true
|
||||||
if resultName == "" {
|
if len(validator.approved) > 0 {
|
||||||
resultName = validator.name
|
index := validator.calls - 1
|
||||||
|
if index >= len(validator.approved) {
|
||||||
|
index = len(validator.approved) - 1
|
||||||
}
|
}
|
||||||
var decisions []contracts.ValidationDecision
|
approved = validator.approved[index]
|
||||||
if validator.decisions != nil {
|
|
||||||
decisions = validator.decisions(req.Candidates)
|
|
||||||
}
|
}
|
||||||
return contracts.ValidationResult{
|
return contracts.ValidationResult{
|
||||||
ValidatorName: resultName,
|
Approved: approved,
|
||||||
Decisions: decisions,
|
ReasonCode: validator.reason,
|
||||||
|
Message: validator.message,
|
||||||
Warnings: validator.warnings,
|
Warnings: validator.warnings,
|
||||||
}, validator.err
|
}, validator.err
|
||||||
}
|
}
|
||||||
@@ -2027,12 +2168,31 @@ func assertRunError(t *testing.T, err error, want string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.RawValidator) *RawValidationRegistry {
|
func resolvedValidatorForTest(validator contracts.Validator) ResolvedValidator {
|
||||||
|
return ResolvedValidator{
|
||||||
|
Binding: Binding(validator.Name()),
|
||||||
|
ExecutionClass: validator.ExecutionClass(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setResolvedValidatorChain(t *testing.T, resolved *ResolvedPipeline, stage ModuleStage, laneID string, module string, validators ...ResolvedValidator) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
registry := NewRawValidationRegistry()
|
if resolved == nil {
|
||||||
if err := registry.Register(stage, module, validators...); err != nil {
|
t.Fatal("resolved pipeline must not be nil")
|
||||||
t.Fatalf("register raw validators: %v", err)
|
|
||||||
}
|
}
|
||||||
return registry
|
chain := ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: laneID,
|
||||||
|
ModuleKey: module,
|
||||||
|
Validators: append([]ResolvedValidator(nil), validators...),
|
||||||
|
}
|
||||||
|
for index := range resolved.ValidatorChains {
|
||||||
|
existing := resolved.ValidatorChains[index]
|
||||||
|
if existing.Stage == stage && existing.LaneID == laneID && existing.ModuleKey == module {
|
||||||
|
resolved.ValidatorChains[index] = chain
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolved.ValidatorChains = append(resolved.ValidatorChains, chain)
|
||||||
}
|
}
|
||||||
|
|||||||
110
internal/framework/pipeline/validator_chain_registry.go
Normal file
110
internal/framework/pipeline/validator_chain_registry.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type validatorChainKey struct {
|
||||||
|
stage ModuleStage
|
||||||
|
module string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorChainMapping struct {
|
||||||
|
Stage ModuleStage `json:"stage"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorChainRegistry struct {
|
||||||
|
chains map[validatorChainKey][]ModuleBinding
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewValidatorChainRegistry() *ValidatorChainRegistry {
|
||||||
|
return &ValidatorChainRegistry{
|
||||||
|
chains: make(map[validatorChainKey][]ModuleBinding),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorChainRegistry) Register(mapping ValidatorChainMapping) error {
|
||||||
|
if r == nil {
|
||||||
|
return fmt.Errorf("validator chain registry must not be nil")
|
||||||
|
}
|
||||||
|
normalized, err := normalizeValidatorChainMapping(mapping)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if r.chains == nil {
|
||||||
|
r.chains = make(map[validatorChainKey][]ModuleBinding)
|
||||||
|
}
|
||||||
|
key := validatorChainKey{stage: normalized.Stage, module: normalized.Module}
|
||||||
|
if _, exists := r.chains[key]; exists {
|
||||||
|
return fmt.Errorf("validator chain for %q %q is already registered", normalized.Stage, normalized.Module)
|
||||||
|
}
|
||||||
|
r.chains[key] = cloneModuleBindings(normalized.Validators)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorChainRegistry) Validators(stage ModuleStage, module string) []ModuleBinding {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
chain := r.chains[validatorChainKey{stage: stage, module: strings.TrimSpace(module)}]
|
||||||
|
return cloneModuleBindings(chain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeValidatorChainMapping(mapping ValidatorChainMapping) (ValidatorChainMapping, error) {
|
||||||
|
normalized := ValidatorChainMapping{
|
||||||
|
Stage: mapping.Stage,
|
||||||
|
Module: strings.TrimSpace(mapping.Module),
|
||||||
|
Validators: cloneModuleBindings(mapping.Validators),
|
||||||
|
}
|
||||||
|
switch normalized.Stage {
|
||||||
|
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||||
|
default:
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain stage %q is not supported", normalized.Stage)
|
||||||
|
}
|
||||||
|
if normalized.Module == "" {
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain module key must not be empty")
|
||||||
|
}
|
||||||
|
for i, validator := range normalized.Validators {
|
||||||
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain for %q %q has empty validator key at index %d", normalized.Stage, normalized.Module, i)
|
||||||
|
}
|
||||||
|
normalized.Validators[i] = resolveBinding(validator, "")
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
|
||||||
|
if len(bindings) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ModuleBinding, len(bindings))
|
||||||
|
for i, binding := range bindings {
|
||||||
|
out[i] = cloneModuleBinding(binding)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||||
|
binding.Module = strings.TrimSpace(binding.Module)
|
||||||
|
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
|
||||||
|
binding.Options = cloneOptions(binding.Options)
|
||||||
|
if len(binding.References) > 0 {
|
||||||
|
references := make(map[string]string, len(binding.References))
|
||||||
|
for key, value := range binding.References {
|
||||||
|
references[key] = value
|
||||||
|
}
|
||||||
|
binding.References = references
|
||||||
|
}
|
||||||
|
binding.Validators = cloneValidatorOverride(binding.Validators)
|
||||||
|
return binding
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
|
||||||
|
return ValidatorOverride{
|
||||||
|
Set: override.Set,
|
||||||
|
Validators: cloneModuleBindings(override.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRegistersAndLooksUpChains(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: " extractor ",
|
||||||
|
Validators: []ModuleBinding{{Module: " first "}, {Module: "second"}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := registry.Validators(StageExtract, " extractor ")
|
||||||
|
if len(got) != 2 || got[0].Module != "first" || got[1].Module != "second" {
|
||||||
|
t.Fatalf("Validators() = %#v, want trimmed chain", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRejectsDuplicateMappings(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
mapping := ValidatorChainMapping{Stage: StageMerge, Module: "merge", Validators: []ModuleBinding{Binding("validator")}}
|
||||||
|
if err := registry.Register(mapping); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := registry.Register(mapping); err == nil {
|
||||||
|
t.Fatal("Register() error = nil, want duplicate mapping error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRejectsUnsupportedStage(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{Stage: StageInput, Module: "input"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Register() error = nil, want unsupported stage error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryAllowsAbsentAndEmptyChains(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||||
|
t.Fatalf("absent chain = %#v, want nil", got)
|
||||||
|
}
|
||||||
|
if err := registry.Register(ValidatorChainMapping{Stage: StageNormalize, Module: "normalize"}); err != nil {
|
||||||
|
t.Fatalf("Register(empty) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||||
|
t.Fatalf("empty chain = %#v, want nil", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryReturnsDefensiveCopies(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageChunk,
|
||||||
|
Module: "chunk",
|
||||||
|
Validators: []ModuleBinding{{Module: "validator", Options: map[string]any{"level": "strict"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := registry.Validators(StageChunk, "chunk")
|
||||||
|
got[0].Module = "changed"
|
||||||
|
got[0].Options["level"] = "changed"
|
||||||
|
|
||||||
|
again := registry.Validators(StageChunk, "chunk")
|
||||||
|
if again[0].Module != "validator" || again[0].Options["level"] != "strict" {
|
||||||
|
t.Fatalf("Validators() after caller mutation = %#v, want original chain", again)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
@@ -9,29 +10,34 @@ import (
|
|||||||
|
|
||||||
type ValidatorConstructor func() (contracts.Validator, error)
|
type ValidatorConstructor func() (contracts.Validator, error)
|
||||||
|
|
||||||
|
type ValidatorSpec struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type ValidatorRegistry struct {
|
type ValidatorRegistry struct {
|
||||||
constructors map[string]ValidatorConstructor
|
constructors map[string]ValidatorConstructor
|
||||||
specs map[string]ModuleSpec
|
specs map[string]ValidatorSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewValidatorRegistry() *ValidatorRegistry {
|
func NewValidatorRegistry() *ValidatorRegistry {
|
||||||
return &ValidatorRegistry{
|
return &ValidatorRegistry{
|
||||||
constructors: make(map[string]ValidatorConstructor),
|
constructors: make(map[string]ValidatorConstructor),
|
||||||
specs: make(map[string]ModuleSpec),
|
specs: make(map[string]ValidatorSpec),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
|
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
|
||||||
return r.RegisterWithSpec(defaultModuleSpec(key, StageValidate), constructor)
|
return r.RegisterWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ValidatorConstructor) error {
|
func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor ValidatorConstructor) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("validator registry must not be nil")
|
return fmt.Errorf("validator registry must not be nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedSpec := normalizeModuleSpec(spec)
|
normalizedSpec, err := normalizeValidatorSpec(spec)
|
||||||
if err := validateModuleSpec("validator", StageValidate, normalizedSpec); err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if constructor == nil {
|
if constructor == nil {
|
||||||
@@ -45,10 +51,10 @@ func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor Valida
|
|||||||
r.constructors = make(map[string]ValidatorConstructor)
|
r.constructors = make(map[string]ValidatorConstructor)
|
||||||
}
|
}
|
||||||
if r.specs == nil {
|
if r.specs == nil {
|
||||||
r.specs = make(map[string]ModuleSpec)
|
r.specs = make(map[string]ValidatorSpec)
|
||||||
}
|
}
|
||||||
r.constructors[normalizedSpec.Key] = constructor
|
r.constructors[normalizedSpec.Key] = constructor
|
||||||
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
r.specs[normalizedSpec.Key] = normalizedSpec
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,20 +83,45 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
|
|||||||
if validator.Name() != normalizedKey {
|
if validator.Name() != normalizedKey {
|
||||||
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
|
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
|
||||||
}
|
}
|
||||||
|
spec, ok := r.specs[normalizedKey]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("validator %q spec is not registered", normalizedKey)
|
||||||
|
}
|
||||||
|
if validator.ExecutionClass() != spec.ExecutionClass {
|
||||||
|
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
|
||||||
|
}
|
||||||
|
|
||||||
return validator, nil
|
return validator, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) Spec(key string) (ModuleSpec, bool) {
|
func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return ModuleSpec{}, false
|
return ValidatorSpec{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
spec, ok := r.specs[strings.TrimSpace(key)]
|
spec, ok := r.specs[strings.TrimSpace(key)]
|
||||||
if !ok {
|
if !ok {
|
||||||
return ModuleSpec{}, false
|
return ValidatorSpec{}, false
|
||||||
}
|
}
|
||||||
return cloneModuleSpec(spec), true
|
return spec, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
|
||||||
|
if r == nil || len(r.specs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(r.specs))
|
||||||
|
for key := range r.specs {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
specs := make([]ValidatorSpec, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
specs = append(specs, r.specs[key])
|
||||||
|
}
|
||||||
|
return specs
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) RegisteredKeys() []string {
|
func (r *ValidatorRegistry) RegisteredKeys() []string {
|
||||||
@@ -100,3 +131,19 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
|
|||||||
|
|
||||||
return sortedRegistryKeys(r.constructors)
|
return sortedRegistryKeys(r.constructors)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
||||||
|
normalized := ValidatorSpec{
|
||||||
|
Key: strings.TrimSpace(spec.Key),
|
||||||
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
}
|
||||||
|
if normalized.Key == "" {
|
||||||
|
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
|
||||||
|
}
|
||||||
|
switch normalized.ExecutionClass {
|
||||||
|
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
|
||||||
|
default:
|
||||||
|
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,58 +1,117 @@
|
|||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidatorRegistryBehavior(t *testing.T) {
|
func TestValidatorRegistryBehavior(t *testing.T) {
|
||||||
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Validator]{
|
registry := NewValidatorRegistry()
|
||||||
name: "ValidatorRegistry",
|
if err := registry.Register(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
|
||||||
key: "generic-validator",
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
stage: StageValidate,
|
}
|
||||||
wrongStage: StageExtract,
|
|
||||||
newRegistry: func() any {
|
validator, err := registry.Build("generic-validator")
|
||||||
return NewValidatorRegistry()
|
if err != nil {
|
||||||
},
|
t.Fatalf("Build() error = %v, want nil", err)
|
||||||
register: func(registry any, key string, constructor func() (contracts.Validator, error)) error {
|
}
|
||||||
return registry.(*ValidatorRegistry).Register(key, constructor)
|
if validator.Name() != "generic-validator" {
|
||||||
},
|
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
|
||||||
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Validator, error)) error {
|
}
|
||||||
return registry.(*ValidatorRegistry).RegisterWithSpec(spec, constructor)
|
|
||||||
},
|
spec, ok := registry.Spec(" generic-validator ")
|
||||||
build: func(registry any, key string) (contracts.Validator, error) {
|
if !ok {
|
||||||
return registry.(*ValidatorRegistry).Build(key)
|
t.Fatal("Spec() ok = false, want true")
|
||||||
},
|
}
|
||||||
spec: func(registry any, key string) (ModuleSpec, bool) {
|
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||||
return registry.(*ValidatorRegistry).Spec(key)
|
if !reflect.DeepEqual(spec, want) {
|
||||||
},
|
t.Fatalf("Spec() = %#v, want %#v", spec, want)
|
||||||
registeredKeys: func(registry any) []string {
|
}
|
||||||
return registry.(*ValidatorRegistry).RegisteredKeys()
|
}
|
||||||
},
|
|
||||||
nilRegister: func(key string, constructor func() (contracts.Validator, error)) error {
|
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
|
||||||
var registry *ValidatorRegistry
|
registry := NewValidatorRegistry()
|
||||||
return registry.Register(key, constructor)
|
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||||
},
|
if err := registry.RegisterWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
|
||||||
nilBuild: func(key string) (contracts.Validator, error) {
|
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||||
var registry *ValidatorRegistry
|
}
|
||||||
return registry.Build(key)
|
|
||||||
},
|
got, ok := registry.Spec("llm-validator")
|
||||||
nilSpec: func(key string) (ModuleSpec, bool) {
|
if !ok {
|
||||||
var registry *ValidatorRegistry
|
t.Fatal("Spec() ok = false, want true")
|
||||||
return registry.Spec(key)
|
}
|
||||||
},
|
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||||
nilRegisteredKey: func() []string {
|
if !reflect.DeepEqual(got, want) {
|
||||||
var registry *ValidatorRegistry
|
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||||
return registry.RegisteredKeys()
|
}
|
||||||
},
|
}
|
||||||
constructor: func(key string) func() (contracts.Validator, error) {
|
|
||||||
|
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
|
||||||
|
registry := NewValidatorRegistry()
|
||||||
|
for _, key := range []string{"zeta", "alpha"} {
|
||||||
|
if err := registry.Register(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
|
||||||
|
t.Fatalf("Register(%q) error = %v", key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
specs := registry.RegisteredSpecs()
|
||||||
|
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
|
||||||
|
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
|
||||||
|
registry := NewValidatorRegistry()
|
||||||
|
err := registry.RegisterWithSpec(
|
||||||
|
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
|
||||||
|
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
|
||||||
|
registry := NewValidatorRegistry()
|
||||||
|
if err := registry.RegisterWithSpec(
|
||||||
|
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||||
|
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := registry.Build("validator")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Build() error = nil, want execution class mismatch")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "execution class") {
|
||||||
|
t.Fatalf("Build() error = %q, want execution class context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type testValidator struct {
|
||||||
|
name string
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatorConstructor(name string, executionClass contracts.ExecutionClass) ValidatorConstructor {
|
||||||
return func() (contracts.Validator, error) {
|
return func() (contracts.Validator, error) {
|
||||||
return registryValidator{name: key}, nil
|
return testValidator{name: name, executionClass: executionClass}, nil
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
moduleKey: func(module contracts.Validator) string {
|
|
||||||
return module.Name()
|
func (validator testValidator) Name() string {
|
||||||
},
|
return validator.name
|
||||||
})
|
}
|
||||||
|
|
||||||
|
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return validator.executionClass
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
|
|||||||
Extractors: NewExtractorRegistry(),
|
Extractors: NewExtractorRegistry(),
|
||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
|
ValidatorChains: NewValidatorChainRegistry(),
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
|
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
package validate
|
package validate
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -12,53 +10,18 @@ const (
|
|||||||
ReasonApproved = "approved"
|
ReasonApproved = "approved"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Approved(candidateIndex int) contracts.ValidationDecision {
|
func Approved() contracts.ValidationResult {
|
||||||
return contracts.ValidationDecision{
|
return contracts.ValidationResult{
|
||||||
CandidateIndex: candidateIndex,
|
|
||||||
Approved: true,
|
Approved: true,
|
||||||
ReasonCode: ReasonApproved,
|
ReasonCode: ReasonApproved,
|
||||||
Message: ReasonApproved,
|
Message: ReasonApproved,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision {
|
func Rejected(reasonCode string, message string) contracts.ValidationResult {
|
||||||
return contracts.ValidationDecision{
|
return contracts.ValidationResult{
|
||||||
CandidateIndex: candidateIndex,
|
|
||||||
Approved: false,
|
Approved: false,
|
||||||
ReasonCode: strings.TrimSpace(reasonCode),
|
ReasonCode: strings.TrimSpace(reasonCode),
|
||||||
Message: strings.TrimSpace(message),
|
Message: strings.TrimSpace(message),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error {
|
|
||||||
if len(candidates) != len(decisions) {
|
|
||||||
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
|
|
||||||
}
|
|
||||||
|
|
||||||
expected := make(map[int]struct{}, len(candidates))
|
|
||||||
for _, candidate := range candidates {
|
|
||||||
if _, ok := expected[candidate.Index]; ok {
|
|
||||||
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
|
|
||||||
}
|
|
||||||
expected[candidate.Index] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := make(map[int]struct{}, len(decisions))
|
|
||||||
for _, decision := range decisions {
|
|
||||||
if _, ok := expected[decision.CandidateIndex]; !ok {
|
|
||||||
return fmt.Errorf("validator returned decision for unknown candidate index %d", decision.CandidateIndex)
|
|
||||||
}
|
|
||||||
if _, ok := seen[decision.CandidateIndex]; ok {
|
|
||||||
return fmt.Errorf("validator returned duplicate decision for candidate index %d", decision.CandidateIndex)
|
|
||||||
}
|
|
||||||
seen[decision.CandidateIndex] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
for candidateIndex := range expected {
|
|
||||||
if _, ok := seen[candidateIndex]; !ok {
|
|
||||||
return fmt.Errorf("validator did not return decision for candidate index %d", candidateIndex)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,105 +1,40 @@
|
|||||||
package validate
|
package validate
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestApproved(t *testing.T) {
|
func TestApproved(t *testing.T) {
|
||||||
decision := Approved(7)
|
result := Approved()
|
||||||
|
|
||||||
if decision.CandidateIndex != 7 {
|
if !result.Approved {
|
||||||
t.Fatalf("CandidateIndex = %d, want 7", decision.CandidateIndex)
|
|
||||||
}
|
|
||||||
if !decision.Approved {
|
|
||||||
t.Fatal("Approved = false, want true")
|
t.Fatal("Approved = false, want true")
|
||||||
}
|
}
|
||||||
if decision.ReasonCode != ReasonApproved {
|
if result.ReasonCode != ReasonApproved {
|
||||||
t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, ReasonApproved)
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonApproved)
|
||||||
}
|
}
|
||||||
if decision.Message != "approved" {
|
if result.Message != "approved" {
|
||||||
t.Fatalf("Message = %q, want approved", decision.Message)
|
t.Fatalf("Message = %q, want approved", result.Message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRejectedTrimsReasonAndMessage(t *testing.T) {
|
func TestRejectedTrimsReasonAndMessage(t *testing.T) {
|
||||||
decision := Rejected(3, " invalid ", "\tmessage\n")
|
result := Rejected(" invalid ", "\tmessage\n")
|
||||||
|
|
||||||
if decision.CandidateIndex != 3 {
|
if result.Approved {
|
||||||
t.Fatalf("CandidateIndex = %d, want 3", decision.CandidateIndex)
|
|
||||||
}
|
|
||||||
if decision.Approved {
|
|
||||||
t.Fatal("Approved = true, want false")
|
t.Fatal("Approved = true, want false")
|
||||||
}
|
}
|
||||||
if decision.ReasonCode != "invalid" {
|
if result.ReasonCode != "invalid" {
|
||||||
t.Fatalf("ReasonCode = %q, want invalid", decision.ReasonCode)
|
t.Fatalf("ReasonCode = %q, want invalid", result.ReasonCode)
|
||||||
}
|
}
|
||||||
if decision.Message != "message" {
|
if result.Message != "message" {
|
||||||
t.Fatalf("Message = %q, want message", decision.Message)
|
t.Fatalf("Message = %q, want message", result.Message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
|
func TestHelpersReturnValidationResults(t *testing.T) {
|
||||||
candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}}
|
var _ contracts.ValidationResult = Approved()
|
||||||
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
|
var _ contracts.ValidationResult = Rejected("reason", "message")
|
||||||
|
|
||||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
|
||||||
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) {
|
|
||||||
if err := EnforceDecisionCardinality(nil, nil); err != nil {
|
|
||||||
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
|
||||||
err := EnforceDecisionCardinality(
|
|
||||||
[]artifacts.ArtifactCandidate{{Index: 1}},
|
|
||||||
[]contracts.ValidationDecision{Approved(2)},
|
|
||||||
)
|
|
||||||
|
|
||||||
assertCardinalityError(t, err, "unknown candidate index 2")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
|
||||||
err := EnforceDecisionCardinality(
|
|
||||||
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
|
||||||
)
|
|
||||||
|
|
||||||
assertCardinalityError(t, err, "duplicate decision")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
|
||||||
err := EnforceDecisionCardinality(
|
|
||||||
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
|
||||||
[]contracts.ValidationDecision{Approved(1)},
|
|
||||||
)
|
|
||||||
|
|
||||||
assertCardinalityError(t, err, "1 decisions for 2 candidates")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
|
|
||||||
err := EnforceDecisionCardinality(
|
|
||||||
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}},
|
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
|
||||||
)
|
|
||||||
|
|
||||||
assertCardinalityError(t, err, "candidate index 1 is duplicated")
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertCardinalityError(t *testing.T, err error, want string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("EnforceDecisionCardinality() error = nil, want error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("EnforceDecisionCardinality() error = %q, want substring %q", err.Error(), want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -217,6 +217,7 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -115,12 +115,17 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
|
|||||||
return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
|
return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ExtractionResult{}, extractorErrorf("load response schema: %w", err)
|
||||||
|
}
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Output: contracts.ExtractOutput{
|
Output: contracts.ExtractOutput{
|
||||||
Schema: contracts.ResponseSchema{
|
Schema: contracts.ResponseSchema{
|
||||||
ID: ResponseSchemaID,
|
ID: ResponseSchemaID,
|
||||||
Name: ResponseSchemaName,
|
Name: ResponseSchemaName,
|
||||||
Version: SchemaVersion,
|
Version: SchemaVersion,
|
||||||
|
JSONSchema: append([]byte(nil), schema.JSONSchema...),
|
||||||
},
|
},
|
||||||
Payload: contracts.RawPayload{
|
Payload: contracts.RawPayload{
|
||||||
Content: content,
|
Content: content,
|
||||||
|
|||||||
@@ -60,6 +60,9 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) {
|
|||||||
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
|
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
|
||||||
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
|
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
|
||||||
}
|
}
|
||||||
|
if !json.Valid(result.Output.Schema.JSONSchema) {
|
||||||
|
t.Fatalf("schema JSON is invalid or missing: %s", result.Output.Schema.JSONSchema)
|
||||||
|
}
|
||||||
if got := string(result.Output.Payload.Content); got != string(client.content) {
|
if got := string(result.Output.Payload.Content); got != string(client.content) {
|
||||||
t.Fatalf("content = %q, want exact raw completion content", got)
|
t.Fatalf("content = %q, want exact raw completion content", got)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,168 +0,0 @@
|
|||||||
package spells
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
shapeValidatorName = "dnd/spells/shape"
|
|
||||||
sourceRefValidatorName = "dnd/spells/source_refs"
|
|
||||||
|
|
||||||
reasonInvalidPayload = "invalid_payload"
|
|
||||||
reasonMissingRequiredField = "missing_required_field"
|
|
||||||
reasonMissingSourceRef = "missing_source_ref"
|
|
||||||
reasonInvalidSourceRef = "invalid_source_ref"
|
|
||||||
reasonSpellNotNearSource = "spell_not_near_source"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ contracts.Validator = ShapeValidator{}
|
|
||||||
var _ contracts.Validator = SourceRefValidator{}
|
|
||||||
|
|
||||||
type ShapeValidator struct{}
|
|
||||||
|
|
||||||
func (validator ShapeValidator) Name() string {
|
|
||||||
return shapeValidatorName
|
|
||||||
}
|
|
||||||
|
|
||||||
func (validator ShapeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
|
||||||
for _, candidate := range req.Candidates {
|
|
||||||
decisions = append(decisions, validateShape(candidate))
|
|
||||||
}
|
|
||||||
return contracts.ValidationResult{
|
|
||||||
ValidatorName: validator.Name(),
|
|
||||||
Decisions: decisions,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type SourceRefValidator struct{}
|
|
||||||
|
|
||||||
func (validator SourceRefValidator) Name() string {
|
|
||||||
return sourceRefValidatorName
|
|
||||||
}
|
|
||||||
|
|
||||||
func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
|
||||||
if req.Source == nil {
|
|
||||||
return contracts.ValidationResult{}, fmt.Errorf("dnd spells source refs validator: source must not be nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
|
||||||
var warnings []contracts.Warning
|
|
||||||
for _, candidate := range req.Candidates {
|
|
||||||
decisions = append(decisions, validateSourceRefs(req.Source, candidate))
|
|
||||||
warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...)
|
|
||||||
}
|
|
||||||
return contracts.ValidationResult{
|
|
||||||
ValidatorName: validator.Name(),
|
|
||||||
Decisions: decisions,
|
|
||||||
Warnings: warnings,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateShape(candidate artifacts.ArtifactCandidate) contracts.ValidationDecision {
|
|
||||||
var payload SpellCast
|
|
||||||
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
|
|
||||||
return validate.Rejected(candidate.Index, reasonInvalidPayload, fmt.Sprintf("invalid spell cast payload: %v", err))
|
|
||||||
}
|
|
||||||
for _, field := range requiredSpellCastFields(payload) {
|
|
||||||
if strings.TrimSpace(field.value) == "" {
|
|
||||||
return validate.Rejected(candidate.Index, reasonMissingRequiredField, fmt.Sprintf("missing required field %q", field.name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return validate.Approved(candidate.Index)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) contracts.ValidationDecision {
|
|
||||||
if len(candidate.SourceRefs) == 0 {
|
|
||||||
return validate.Rejected(candidate.Index, reasonMissingSourceRef, "spell cast candidate must include at least one source ref")
|
|
||||||
}
|
|
||||||
for _, ref := range candidate.SourceRefs {
|
|
||||||
if err := source.ValidateRef(doc, ref); err != nil {
|
|
||||||
return validate.Rejected(candidate.Index, reasonInvalidSourceRef, err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return validate.Approved(candidate.Index)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning {
|
|
||||||
if doc == nil || len(candidate.SourceRefs) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var payload SpellCast
|
|
||||||
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
spell := strings.TrimSpace(payload.Spell)
|
|
||||||
if spell == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
needle := strings.ToLower(spell)
|
|
||||||
for _, ref := range candidate.SourceRefs {
|
|
||||||
text, ok := sourceRefText(doc, ref)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if strings.Contains(strings.ToLower(text), needle) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return []contracts.Warning{
|
|
||||||
{
|
|
||||||
Scope: fmt.Sprintf("candidate.%d", candidate.Index),
|
|
||||||
ReasonCode: reasonSpellNotNearSource,
|
|
||||||
Message: fmt.Sprintf("spell %q was not found in the cited source text", spell),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
|
|
||||||
if err := source.ValidateRef(doc, ref); err != nil {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
start := -1
|
|
||||||
end := -1
|
|
||||||
for i, unit := range doc.Units {
|
|
||||||
if unit.ID == ref.StartUnitID {
|
|
||||||
start = i
|
|
||||||
}
|
|
||||||
if unit.ID == ref.EndUnitID {
|
|
||||||
end = i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if start < 0 || end < start {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
var b strings.Builder
|
|
||||||
for i := start; i <= end; i++ {
|
|
||||||
if b.Len() > 0 {
|
|
||||||
b.WriteString("\n")
|
|
||||||
}
|
|
||||||
b.WriteString(doc.Units[i].Text)
|
|
||||||
}
|
|
||||||
return b.String(), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func requiredSpellCastFields(payload SpellCast) []struct {
|
|
||||||
name string
|
|
||||||
value string
|
|
||||||
} {
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
value string
|
|
||||||
}{
|
|
||||||
{name: "caster", value: payload.Caster},
|
|
||||||
{name: "spell", value: payload.Spell},
|
|
||||||
{name: "effect", value: payload.Effect},
|
|
||||||
{name: "narrative_description", value: payload.NarrativeDescription},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,277 +0,0 @@
|
|||||||
package spells
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestValidatorsApproveValidCandidate(t *testing.T) {
|
|
||||||
candidate := validSpellCandidate(7)
|
|
||||||
|
|
||||||
shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, shapeResult, shapeValidatorName, 7, true, validate.ReasonApproved)
|
|
||||||
|
|
||||||
sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Source: promptSourceDocument(),
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved)
|
|
||||||
if len(sourceRefResult.Warnings) != 0 {
|
|
||||||
t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestShapeValidatorRejectsMalformedPayload(t *testing.T) {
|
|
||||||
candidate := validSpellCandidate(3)
|
|
||||||
candidate.Payload = json.RawMessage(`{"caster":`)
|
|
||||||
|
|
||||||
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, shapeValidatorName, 3, false, reasonInvalidPayload)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestShapeValidatorRejectsBlankRequiredFields(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(*SpellCast)
|
|
||||||
}{
|
|
||||||
{name: "caster", mutate: func(payload *SpellCast) { payload.Caster = " \t" }},
|
|
||||||
{name: "spell", mutate: func(payload *SpellCast) { payload.Spell = "" }},
|
|
||||||
{name: "effect", mutate: func(payload *SpellCast) { payload.Effect = "\n" }},
|
|
||||||
{name: "narrative description", mutate: func(payload *SpellCast) { payload.NarrativeDescription = " " }},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
payload := validSpellPayload()
|
|
||||||
tt.mutate(&payload)
|
|
||||||
candidate := validSpellCandidate(5)
|
|
||||||
candidate.Payload = mustSpellPayload(t, payload)
|
|
||||||
|
|
||||||
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, shapeValidatorName, 5, false, reasonMissingRequiredField)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestShapeValidatorDoesNotRequireSourceDocument(t *testing.T) {
|
|
||||||
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(11)},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, shapeValidatorName, 11, true, validate.ReasonApproved)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceRefValidatorRejectsMissingRefs(t *testing.T) {
|
|
||||||
candidate := validSpellCandidate(13)
|
|
||||||
candidate.SourceRefs = nil
|
|
||||||
|
|
||||||
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Source: promptSourceDocument(),
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, sourceRefValidatorName, 13, false, reasonMissingSourceRef)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
ref source.SourceRef
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "unknown source id",
|
|
||||||
ref: source.SourceRef{SourceID: "session-beta", StartUnitID: 1, EndUnitID: 2},
|
|
||||||
want: "does not match",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unknown start unit",
|
|
||||||
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 999, EndUnitID: 2},
|
|
||||||
want: "start_unit_id",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unknown end unit",
|
|
||||||
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 999},
|
|
||||||
want: "end_unit_id",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "reversed unit range",
|
|
||||||
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 1},
|
|
||||||
want: "appears after",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
candidate := validSpellCandidate(17)
|
|
||||||
candidate.SourceRefs = []source.SourceRef{tt.ref}
|
|
||||||
|
|
||||||
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Source: promptSourceDocument(),
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, sourceRefValidatorName, 17, false, reasonInvalidSourceRef)
|
|
||||||
if !strings.Contains(result.Decisions[0].Message, tt.want) {
|
|
||||||
t.Fatalf("Message = %q, want substring %q", result.Decisions[0].Message, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) {
|
|
||||||
candidate := validSpellCandidate(31)
|
|
||||||
payload := validSpellPayload()
|
|
||||||
payload.Spell = "Shield"
|
|
||||||
candidate.Payload = mustSpellPayload(t, payload)
|
|
||||||
|
|
||||||
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Source: promptSourceDocument(),
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved)
|
|
||||||
if len(result.Warnings) != 1 {
|
|
||||||
t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings)
|
|
||||||
}
|
|
||||||
warning := result.Warnings[0]
|
|
||||||
if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") {
|
|
||||||
t.Fatalf("warning = %#v, want spell relatedness warning", warning)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) {
|
|
||||||
_, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("SourceRefValidator.Validate() error = nil, want source error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "source") {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %q, want source context", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidatorsPreserveCandidateIndexes(t *testing.T) {
|
|
||||||
candidates := []artifacts.ArtifactCandidate{
|
|
||||||
validSpellCandidate(23),
|
|
||||||
validSpellCandidate(29),
|
|
||||||
}
|
|
||||||
|
|
||||||
shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Candidates: candidates,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertDecisionIndexes(t, shapeResult.Decisions, []int{23, 29})
|
|
||||||
|
|
||||||
sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
|
|
||||||
Source: promptSourceDocument(),
|
|
||||||
Candidates: candidates,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
assertDecisionIndexes(t, sourceRefResult.Decisions, []int{23, 29})
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSpellCandidate(index int) artifacts.ArtifactCandidate {
|
|
||||||
return artifacts.ArtifactCandidate{
|
|
||||||
Index: index,
|
|
||||||
Payload: spellPayload(validSpellPayload()),
|
|
||||||
SourceRefs: []source.SourceRef{
|
|
||||||
{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSpellPayload() SpellCast {
|
|
||||||
return SpellCast{
|
|
||||||
Caster: "Aria",
|
|
||||||
Spell: "Cure Wounds",
|
|
||||||
Effect: "Heals an injured ally.",
|
|
||||||
NarrativeDescription: "Aria restores the fighter after the fight.",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustSpellPayload(t *testing.T, payload SpellCast) json.RawMessage {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return spellPayload(payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
func spellPayload(payload SpellCast) json.RawMessage {
|
|
||||||
encoded, err := json.Marshal(payload)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return encoded
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertSingleDecision(t *testing.T, result contracts.ValidationResult, wantName string, wantIndex int, wantApproved bool, wantReason string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if result.ValidatorName != wantName {
|
|
||||||
t.Fatalf("ValidatorName = %q, want %q", result.ValidatorName, wantName)
|
|
||||||
}
|
|
||||||
if len(result.Decisions) != 1 {
|
|
||||||
t.Fatalf("len(Decisions) = %d, want 1", len(result.Decisions))
|
|
||||||
}
|
|
||||||
decision := result.Decisions[0]
|
|
||||||
if decision.CandidateIndex != wantIndex {
|
|
||||||
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, wantIndex)
|
|
||||||
}
|
|
||||||
if decision.Approved != wantApproved {
|
|
||||||
t.Fatalf("Approved = %t, want %t", decision.Approved, wantApproved)
|
|
||||||
}
|
|
||||||
if decision.ReasonCode != wantReason {
|
|
||||||
t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, wantReason)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertDecisionIndexes(t *testing.T, decisions []contracts.ValidationDecision, want []int) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if len(decisions) != len(want) {
|
|
||||||
t.Fatalf("len(Decisions) = %d, want %d", len(decisions), len(want))
|
|
||||||
}
|
|
||||||
for i := range want {
|
|
||||||
if decisions[i].CandidateIndex != want[i] {
|
|
||||||
t.Fatalf("Decisions[%d].CandidateIndex = %d, want %d", i, decisions[i].CandidateIndex, want[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -156,6 +156,7 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -189,18 +189,28 @@ func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
|
|||||||
}
|
}
|
||||||
schema := outputs[0].Schema
|
schema := outputs[0].Schema
|
||||||
for _, output := range outputs[1:] {
|
for _, output := range outputs[1:] {
|
||||||
if output.Schema != schema {
|
if !sameResponseSchema(output.Schema, schema) {
|
||||||
return contracts.ResponseSchema{}
|
return contracts.ResponseSchema{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return schema
|
return schema
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
|
||||||
|
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
|
||||||
|
}
|
||||||
|
|
||||||
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
||||||
|
output.Schema = cloneResponseSchema(output.Schema)
|
||||||
output.Payload = cloneRawPayload(output.Payload)
|
output.Payload = cloneRawPayload(output.Payload)
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||||
|
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
|
||||||
|
return schema
|
||||||
|
}
|
||||||
|
|
||||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||||
return contracts.RawPayload{
|
return contracts.RawPayload{
|
||||||
Content: append([]byte(nil), payload.Content...),
|
Content: append([]byte(nil), payload.Content...),
|
||||||
|
|||||||
60
internal/validators/extract/dnd/spells/shape/validator.go
Normal file
60
internal/validators/extract/dnd/spells/shape/validator.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package shape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/spellpayload"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "extract/dnd/spells/shape"
|
||||||
|
const ReasonCode = "invalid_spell_shape"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
payload, err := spellpayload.ValidationRequestPayload(req)
|
||||||
|
if err != nil {
|
||||||
|
return rejection(err.Error()), nil
|
||||||
|
}
|
||||||
|
if err := spellpayload.ValidateShape(payload); err != nil {
|
||||||
|
return rejection(err.Error()), nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejection(message string) contracts.ValidationResult {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCode,
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package shape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorApprovesWellFormedSpellPayload(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"heals","narrative_description":"Aria heals Borin.","source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Validate() = %#v, want approved", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsMalformedPayload(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(`{"spell_casts":`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsMissingRequiredSpellFields(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(`{"spell_casts":[{"caster":"Aria","effect":"heals","narrative_description":"Aria heals Borin.","source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("validator = %q/%q, want key and deterministic execution", validator.Name(), validator.ExecutionClass())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithPayload(payload string) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Payload: contracts.RawPayload{
|
||||||
|
Content: []byte(payload),
|
||||||
|
MediaType: "application/json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package sourcerefs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/spellpayload"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "extract/dnd/spells/source_refs"
|
||||||
|
const ReasonCode = "invalid_source_refs"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
payload, err := spellpayload.ValidationRequestPayload(req)
|
||||||
|
if err != nil {
|
||||||
|
return rejection(err.Error()), nil
|
||||||
|
}
|
||||||
|
if err := spellpayload.ValidateShape(payload); err != nil {
|
||||||
|
return rejection(err.Error()), nil
|
||||||
|
}
|
||||||
|
for spellIndex, spell := range payload.SpellCasts {
|
||||||
|
for refIndex, ref := range spellpayload.SourceRefCandidates(req.Source, spell) {
|
||||||
|
if err := source.ValidateRef(req.Source, ref); err != nil {
|
||||||
|
return rejection(fmt.Sprintf("spell_casts[%d].source_refs[%d]: %v", spellIndex, refIndex, err)), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejection(message string) contracts.ValidationResult {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCode,
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package sourcerefs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorApprovesValidSourceRefs(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(validDocument(), `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"heals","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":2}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Validate() = %#v, want approved", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsInvalidSourceRefs(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(validDocument(), `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"heals","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"source_id":"session","start_unit_id":99,"end_unit_id":99}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsMissingSourceDocument(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(nil, `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"heals","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("validator = %q/%q, want key and deterministic execution", validator.Name(), validator.ExecutionClass())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithPayload(doc *source.SourceDocument, payload string) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Source: doc,
|
||||||
|
Payload: contracts.RawPayload{
|
||||||
|
Content: []byte(payload),
|
||||||
|
MediaType: "application/json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validDocument() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "session",
|
||||||
|
Kind: "transcript",
|
||||||
|
Format: "application/json",
|
||||||
|
Digest: "sha256:session",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{ID: 1, Kind: "message", Text: "Aria raises her holy symbol."},
|
||||||
|
{ID: 2, Kind: "message", Text: "Aria casts Cure Wounds on Borin."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package sourcerelatedness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/spellpayload"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "extract/dnd/spells/source_relatedness"
|
||||||
|
const WarningReasonCode = "spell_not_near_source"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
payload, err := spellpayload.ValidationRequestPayload(req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
if err := spellpayload.ValidateShape(payload); err != nil {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var warnings []contracts.Warning
|
||||||
|
for spellIndex, spell := range payload.SpellCasts {
|
||||||
|
if !spellAppearsInCitedText(req.Source, spell) {
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: fmt.Sprintf("spell_casts[%d]", spellIndex),
|
||||||
|
ReasonCode: WarningReasonCode,
|
||||||
|
Message: fmt.Sprintf("spell %q was not found in cited source text", strings.TrimSpace(spell.Spell)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func spellAppearsInCitedText(doc *source.SourceDocument, spell spellpayload.SpellCast) bool {
|
||||||
|
name := strings.ToLower(strings.TrimSpace(spell.Spell))
|
||||||
|
if name == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, ref := range spellpayload.SourceRefCandidates(doc, spell) {
|
||||||
|
text, ok := spellpayload.CitedText(doc, ref)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(text), name) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package sourcerelatedness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorApprovesWithoutWarningWhenSpellAppearsInCitedText(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(validDocument(), `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"heals","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"source_id":"session","start_unit_id":2,"end_unit_id":2}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Approved = false, want true")
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 0 {
|
||||||
|
t.Fatalf("Warnings = %#v, want none", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorWarnsWhenSpellDoesNotAppearInCitedText(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(validDocument(), `{"spell_casts":[{"caster":"Borin","spell":"Fire Bolt","effect":"scorches","narrative_description":"Borin casts Fire Bolt.","source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Approved = false, want true")
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 {
|
||||||
|
t.Fatalf("Warnings = %#v, want one warning", result.Warnings)
|
||||||
|
}
|
||||||
|
if result.Warnings[0].ReasonCode != WarningReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.Warnings[0].ReasonCode, WarningReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorApprovesMalformedPayloadWithoutWarning(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(validDocument(), `{"spell_casts":`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Approved = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("validator = %q/%q, want key and deterministic execution", validator.Name(), validator.ExecutionClass())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithPayload(doc *source.SourceDocument, payload string) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Source: doc,
|
||||||
|
Payload: contracts.RawPayload{
|
||||||
|
Content: []byte(payload),
|
||||||
|
MediaType: "application/json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validDocument() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "session",
|
||||||
|
Kind: "transcript",
|
||||||
|
Format: "application/json",
|
||||||
|
Digest: "sha256:session",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{ID: 1, Kind: "message", Text: "Borin draws his dagger."},
|
||||||
|
{ID: 2, Kind: "message", Text: "Aria casts Cure Wounds on Borin."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package spellpayload
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Payload struct {
|
||||||
|
SpellCasts []SpellCast `json:"spell_casts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SpellCast struct {
|
||||||
|
Caster string `json:"caster"`
|
||||||
|
Spell string `json:"spell"`
|
||||||
|
Effect string `json:"effect"`
|
||||||
|
NarrativeDescription string `json:"narrative_description"`
|
||||||
|
SourceRefs []dnd.SourceRefResponse `json:"source_refs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func Parse(raw []byte) (Payload, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
|
||||||
|
var payload Payload
|
||||||
|
if err := decoder.Decode(&payload); err != nil {
|
||||||
|
return Payload{}, fmt.Errorf("parse spell payload: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return Payload{}, fmt.Errorf("parse spell payload: multiple JSON values")
|
||||||
|
}
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateShape(payload Payload) error {
|
||||||
|
if payload.SpellCasts == nil {
|
||||||
|
return fmt.Errorf("spell_casts must be present")
|
||||||
|
}
|
||||||
|
for index, spell := range payload.SpellCasts {
|
||||||
|
if strings.TrimSpace(spell.Caster) == "" {
|
||||||
|
return fmt.Errorf("spell_casts[%d].caster must not be empty", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(spell.Spell) == "" {
|
||||||
|
return fmt.Errorf("spell_casts[%d].spell must not be empty", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(spell.Effect) == "" {
|
||||||
|
return fmt.Errorf("spell_casts[%d].effect must not be empty", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(spell.NarrativeDescription) == "" {
|
||||||
|
return fmt.Errorf("spell_casts[%d].narrative_description must not be empty", index)
|
||||||
|
}
|
||||||
|
if len(spell.SourceRefs) == 0 {
|
||||||
|
return fmt.Errorf("spell_casts[%d].source_refs must not be empty", index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func SourceRefCandidates(doc *source.SourceDocument, spell SpellCast) []source.SourceRef {
|
||||||
|
refs := make([]source.SourceRef, 0, len(spell.SourceRefs))
|
||||||
|
for _, ref := range spell.SourceRefs {
|
||||||
|
refs = append(refs, dnd.SourceRefCandidate(doc, ref))
|
||||||
|
}
|
||||||
|
return refs
|
||||||
|
}
|
||||||
|
|
||||||
|
func CitedText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
|
||||||
|
if doc == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
startIndex, ok := source.UnitIndex(doc, ref.StartUnitID)
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
endIndex, ok := source.UnitIndex(doc, ref.EndUnitID)
|
||||||
|
if !ok || startIndex > endIndex {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
for i := startIndex; i <= endIndex; i++ {
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
b.WriteString(doc.Units[i].Text)
|
||||||
|
}
|
||||||
|
return b.String(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidationRequestPayload(req contracts.ValidationRequest) (Payload, error) {
|
||||||
|
return Parse(req.Payload.Content)
|
||||||
|
}
|
||||||
43
internal/validators/generic/always_accept/validator.go
Normal file
43
internal/validators/generic/always_accept/validator.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package alwaysaccept
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "generic/always_accept"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
40
internal/validators/generic/always_accept/validator_test.go
Normal file
40
internal/validators/generic/always_accept/validator_test.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
package alwaysaccept
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorApproves(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Approved = false, want true")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != "" || result.Message != "" {
|
||||||
|
t.Fatalf("result = %#v, want approval without rejection details", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key {
|
||||||
|
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
48
internal/validators/generic/always_reject/validator.go
Normal file
48
internal/validators/generic/always_reject/validator.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package alwaysreject
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "generic/always_reject"
|
||||||
|
const ReasonCode = "always_reject"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCode,
|
||||||
|
Message: "output rejected by always-reject validator",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
43
internal/validators/generic/always_reject/validator_test.go
Normal file
43
internal/validators/generic/always_reject/validator_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package alwaysreject
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorRejects(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCode {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
|
||||||
|
}
|
||||||
|
if result.Message == "" {
|
||||||
|
t.Fatal("Message = empty, want rejection message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key {
|
||||||
|
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
52
internal/validators/generic/valid_json/validator.go
Normal file
52
internal/validators/generic/valid_json/validator.go
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
package validjson
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "generic/valid_json"
|
||||||
|
const ReasonCodeInvalidJSON = "invalid_json"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
if !json.Valid(req.Payload.Content) {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeInvalidJSON,
|
||||||
|
Message: "payload is not valid JSON",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
66
internal/validators/generic/valid_json/validator_test.go
Normal file
66
internal/validators/generic/valid_json/validator_test.go
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
package validjson
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorAcceptsValidJSON(t *testing.T) {
|
||||||
|
tests := []string{
|
||||||
|
`{"value":true}`,
|
||||||
|
`[1,2,3]`,
|
||||||
|
`"value"`,
|
||||||
|
}
|
||||||
|
for _, payload := range tests {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(payload))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate(%s) error = %v, want nil", payload, err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Validate(%s) = %#v, want approved", payload, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsInvalidJSON(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithPayload(`{"value":`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCodeInvalidJSON {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key {
|
||||||
|
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithPayload(payload string) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Payload: contracts.RawPayload{
|
||||||
|
Content: []byte(payload),
|
||||||
|
MediaType: "application/json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
81
internal/validators/generic/valid_json_schema/validator.go
Normal file
81
internal/validators/generic/valid_json_schema/validator.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package validjsonschema
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "generic/valid_json_schema"
|
||||||
|
const ReasonCodeInvalidJSON = "invalid_json"
|
||||||
|
const ReasonCodeSchemaInvalid = "json_schema_invalid"
|
||||||
|
|
||||||
|
var _ contracts.Validator = (*Validator)(nil)
|
||||||
|
|
||||||
|
type Validator struct{}
|
||||||
|
|
||||||
|
func New() *Validator {
|
||||||
|
return &Validator{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
if len(req.Schema.JSONSchema) == 0 {
|
||||||
|
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
|
||||||
|
}
|
||||||
|
|
||||||
|
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Payload.Content))
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeInvalidJSON,
|
||||||
|
Message: "payload is not valid JSON",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Schema.JSONSchema))
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, fmt.Errorf("parse response schema: %w", err)
|
||||||
|
}
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||||
|
return contracts.ValidationResult{}, fmt.Errorf("load response schema: %w", err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile("schema.json")
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, fmt.Errorf("compile response schema: %w", err)
|
||||||
|
}
|
||||||
|
if err := schema.Validate(instance); err != nil {
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeSchemaInvalid,
|
||||||
|
Message: "payload does not conform to response schema",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{
|
||||||
|
Key: Key,
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
111
internal/validators/generic/valid_json_schema/validator_test.go
Normal file
111
internal/validators/generic/valid_json_schema/validator_test.go
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
package validjsonschema
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorAcceptsSchemaConformantJSON(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, objectSchema()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !result.Approved {
|
||||||
|
t.Fatalf("Validate() = %#v, want approved", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsInvalidPayloadJSON(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":`, objectSchema()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCodeInvalidJSON {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsSchemaNonConformance(t *testing.T) {
|
||||||
|
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":3}`, objectSchema()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if result.Approved {
|
||||||
|
t.Fatalf("Approved = true, want false")
|
||||||
|
}
|
||||||
|
if result.ReasonCode != ReasonCodeSchemaInvalid {
|
||||||
|
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeSchemaInvalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorErrorsWhenSchemaContentMissing(t *testing.T) {
|
||||||
|
_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, nil))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want missing schema content error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "schema content") {
|
||||||
|
t.Fatalf("Validate() error = %q, want schema content context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorErrorsWhenSchemaContentIsMalformed(t *testing.T) {
|
||||||
|
_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, []byte(`{"type":`)))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want malformed schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "parse response schema") {
|
||||||
|
t.Fatalf("Validate() error = %q, want parse schema context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecAndRegister(t *testing.T) {
|
||||||
|
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
validator, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if validator.Name() != Key {
|
||||||
|
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithSchema(payload string, schema []byte) contracts.ValidationRequest {
|
||||||
|
return contracts.ValidationRequest{
|
||||||
|
Schema: contracts.ResponseSchema{
|
||||||
|
ID: "test.schema",
|
||||||
|
Name: "test_schema",
|
||||||
|
Version: "v1",
|
||||||
|
JSONSchema: append([]byte(nil), schema...),
|
||||||
|
},
|
||||||
|
Payload: contracts.RawPayload{
|
||||||
|
Content: []byte(payload),
|
||||||
|
MediaType: "application/json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectSchema() []byte {
|
||||||
|
return []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["name"],
|
||||||
|
"properties": {
|
||||||
|
"name": {"type": "string"}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}`)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user