Compare commits
6 Commits
e70450c401
...
9d3175d36a
| Author | SHA1 | Date | |
|---|---|---|---|
| 9d3175d36a | |||
| 4f96abf42c | |||
| d88bcb6070 | |||
| 0cca3b1f5d | |||
| bbc83ab042 | |||
| 2cba6d4512 |
@@ -132,8 +132,8 @@ handling of debug data is defined in [Operations](../operations.md#debug).
|
|||||||
|
|
||||||
## Tests To Inspect
|
## Tests To Inspect
|
||||||
|
|
||||||
- `internal/framework/llm/scriptorium_client_test.go` and
|
- `internal/framework/llm/scriptorium_client_test.go`: adapter mapping and local
|
||||||
`scriptorium_api_test.go`: adapter mapping and local HTTP integration.
|
HTTP integration.
|
||||||
- `internal/framework/llm/scheduler_test.go` and
|
- `internal/framework/llm/scheduler_test.go` and
|
||||||
`scheduled_client_test.go`: permits, FIFO behavior, cancellation, and wrapper
|
`scheduled_client_test.go`: permits, FIFO behavior, cancellation, and wrapper
|
||||||
release.
|
release.
|
||||||
@@ -141,7 +141,7 @@ handling of debug data is defined in [Operations](../operations.md#debug).
|
|||||||
`schema_registry_test.go`: asset composition, validation, and defensive
|
`schema_registry_test.go`: asset composition, validation, and defensive
|
||||||
copies.
|
copies.
|
||||||
- `internal/framework/llm/secrets_test.go`: provider-error redaction.
|
- `internal/framework/llm/secrets_test.go`: provider-error redaction.
|
||||||
- `internal/cli/run_test.go`: profile validation, production client wiring,
|
- `internal/cli/run_contract_test.go`: profile validation, production client
|
||||||
manifest recording, and debug integration.
|
wiring, manifest recording, and debug integration.
|
||||||
- Module-local `scriptorium_assets_test.go` files: prompt inputs and package
|
- Module-local `scriptorium_assets_test.go` files: prompt inputs and package
|
||||||
asset registration.
|
asset registration.
|
||||||
|
|||||||
@@ -238,10 +238,11 @@ does not inventory implementations.
|
|||||||
- Package-local `*_test.go` files under the module or validator being changed.
|
- Package-local `*_test.go` files under the module or validator being changed.
|
||||||
- `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
|
- `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
|
||||||
and heterogeneous artifact composition.
|
and heterogeneous artifact composition.
|
||||||
- `internal/framework/pipeline/default_modules_test.go`: framework binding
|
- `internal/framework/pipeline/profile_test.go`: framework binding defaults and
|
||||||
defaults.
|
profile resolution.
|
||||||
- `internal/cli/run_test.go`: production catalog, config resolution, and
|
- `internal/cli/production_contract_test.go`: production catalog, config
|
||||||
end-to-end CLI composition.
|
resolution, and composition smoke coverage.
|
||||||
|
- `internal/cli/example_contract_test.go`: maintained example ownership.
|
||||||
- `internal/framework/promptfs/*_test.go` and
|
- `internal/framework/promptfs/*_test.go` and
|
||||||
`internal/modules/dnd/shared/*_test.go`: shared prompt and reference assembly.
|
`internal/modules/dnd/shared/*_test.go`: shared prompt and reference assembly.
|
||||||
- `internal/modules/integration/*_test.go`: black-box composition across
|
- `internal/modules/integration/*_test.go`: black-box composition across
|
||||||
|
|||||||
@@ -321,8 +321,12 @@ stage, resolved lane, and source chunk rather than completion time.
|
|||||||
construction order, dependency failures, and the before-source-work boundary.
|
construction order, dependency failures, and the before-source-work boundary.
|
||||||
- `internal/framework/pipeline/references_test.go`: target resolution and
|
- `internal/framework/pipeline/references_test.go`: target resolution and
|
||||||
materialization.
|
materialization.
|
||||||
- `internal/cli/run_test.go`: production stage transitions, retries, rejections,
|
- `internal/cli/run_contract_test.go`: production run transitions, retries,
|
||||||
warnings, debug hooks, manifests, and end-to-end composition.
|
rejections, warnings, debug hooks, and manifests.
|
||||||
|
- `internal/cli/production_contract_test.go`: production composition and
|
||||||
|
configuration-resolution smoke coverage.
|
||||||
|
- `internal/cli/example_contract_test.go`: maintained example resolution and
|
||||||
|
execution ownership.
|
||||||
- `internal/modules/integration/*_test.go` and
|
- `internal/modules/integration/*_test.go` and
|
||||||
`internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
|
`internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
|
||||||
composition across concrete module families.
|
composition across concrete module families.
|
||||||
|
|||||||
@@ -52,10 +52,13 @@ separately and never replace the command's primary error.
|
|||||||
|
|
||||||
## Tests To Inspect
|
## Tests To Inspect
|
||||||
|
|
||||||
- `internal/cli/state_surfaces_test.go`: debug allocation and configuration
|
- `internal/cli/run_contract_test.go`: command-owned state allocation,
|
||||||
boundaries.
|
terminalization, and output/report boundaries.
|
||||||
- `internal/cli/state_hardening_test.go`: independent roots, reuse, failures,
|
- `internal/cli/state_hardening_test.go`: independent roots, reuse, failures,
|
||||||
permissions, cleanup, and redaction.
|
permissions, cleanup, and redaction.
|
||||||
|
- `internal/cli/production_contract_test.go`: production composition and
|
||||||
|
configuration validation at the CLI boundary.
|
||||||
|
- `internal/cli/example_contract_test.go`: maintained example ownership.
|
||||||
- `internal/core/debugbundle/*_test.go`: bundle allocation and summary writes.
|
- `internal/core/debugbundle/*_test.go`: bundle allocation and summary writes.
|
||||||
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and
|
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and
|
||||||
reuse.
|
reuse.
|
||||||
|
|||||||
488
docs/roadmap/implementation.md
Normal file
488
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,488 @@
|
|||||||
|
# Test Suite Policy Review Implementation
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Completed on 2026-07-18 by the commit series ending with `4f96abf`. This
|
||||||
|
document is retained as the execution record for the completed test-suite
|
||||||
|
policy review; its stages are no longer active implementation instructions.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document records the decision-complete execution plan used to implement
|
||||||
|
the target revisions in [Test Suite Policy Review](tests.md). The stages were
|
||||||
|
followed in order. The feature roadmap owns the review evidence, desired test
|
||||||
|
ownership, required dispositions, and target state; this document records the
|
||||||
|
sequencing, file-level work, validation, and stop conditions used during
|
||||||
|
implementation.
|
||||||
|
|
||||||
|
This is test-suite harmonization, not a production feature change. Do not
|
||||||
|
modify production behavior. If a new durable test exposes incorrect production
|
||||||
|
behavior, preserve the failing evidence, stop the affected stage, and report
|
||||||
|
the defect separately for explicit scoping.
|
||||||
|
|
||||||
|
## Governing Policies
|
||||||
|
|
||||||
|
Before implementation, read and follow:
|
||||||
|
|
||||||
|
- [Testing Policy](../policy/testing.md), especially behavioral ownership,
|
||||||
|
semantic duplication, test doubles, exact diagnostics, and deletion criteria;
|
||||||
|
- [Architecture](../policy/architecture.md), especially dependency direction,
|
||||||
|
typed artifact boundaries, validation ownership, checkpoint safety, and
|
||||||
|
source/reference separation; and
|
||||||
|
- [Documentation Policy](../policy/documentation.md), especially canonical
|
||||||
|
ownership, current-versus-future behavior, and maintenance of test-routing
|
||||||
|
links.
|
||||||
|
|
||||||
|
The following constraints apply to every stage:
|
||||||
|
|
||||||
|
- preserve unrelated user changes in a dirty worktree;
|
||||||
|
- make no production-code changes unless separately authorized after reporting
|
||||||
|
a confirmed production defect;
|
||||||
|
- add no coverage-percentage target and do not use line coverage as a success
|
||||||
|
metric;
|
||||||
|
- do not mechanically convert tests to tables, consolidate leaf registration
|
||||||
|
tests, or introduce new test frameworks;
|
||||||
|
- keep the default suite deterministic, offline, credential-free, and safe for
|
||||||
|
repeated and parallel execution;
|
||||||
|
- use real filesystem collaborators with `t.TempDir()` for checkpoint behavior;
|
||||||
|
- assert typed/structured outcomes or stable category fragments rather than
|
||||||
|
complete incidental error wording; and
|
||||||
|
- do not update fixtures automatically or add a golden-update path.
|
||||||
|
|
||||||
|
## Stage 0 - Establish the implementation baseline
|
||||||
|
|
||||||
|
1. Read [Test Suite Policy Review](tests.md) completely, including the retained
|
||||||
|
ownership map and every required finding.
|
||||||
|
2. Inspect the worktree and preserve unrelated changes. Limit planned edits to
|
||||||
|
tests and the internal documentation routing identified below.
|
||||||
|
3. Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./cmd/notarius
|
||||||
|
go test -race ./internal/framework/pipeline ./internal/framework/llm ./internal/cli ./internal/modules/integration
|
||||||
|
```
|
||||||
|
|
||||||
|
4. If the baseline fails for a reason unrelated to the planned work, stop and
|
||||||
|
report it. Do not rewrite expectations merely to obtain a clean baseline.
|
||||||
|
|
||||||
|
Stage 0 is complete when the starting state and any pre-existing worktree
|
||||||
|
changes are recorded and all baseline commands pass.
|
||||||
|
|
||||||
|
## Stage 1 - Protect filesystem checkpoint compatibility and recovery
|
||||||
|
|
||||||
|
Complete this stage before deleting or consolidating existing tests so the
|
||||||
|
highest-risk protection is strengthened first.
|
||||||
|
|
||||||
|
### 1.1 Checkpoint identity
|
||||||
|
|
||||||
|
Create `internal/framework/checkpoint/identity_test.go`. Test through
|
||||||
|
`NewIdentity` and `Identity.RelativePath`, not private normalization helpers.
|
||||||
|
|
||||||
|
Build one representative `pipeline.ResolvedPipeline` with two artifact lanes,
|
||||||
|
an input module, stable pipeline ID/digest, selected lanes, runtime
|
||||||
|
fingerprints, reference provenance, and provenance fingerprints. Add tests that
|
||||||
|
prove:
|
||||||
|
|
||||||
|
- reordering selected lanes, runtime fingerprints, references, provenance
|
||||||
|
fingerprints, or resolved lanes does not change the resulting identity;
|
||||||
|
- duplicate or blank selected-lane entries do not change the normalized lane
|
||||||
|
set, and blank fingerprint entries do not change the identity;
|
||||||
|
- changing each meaningful identity input independently changes the digest:
|
||||||
|
pipeline ID or digest, input key, raw/source digest, selected lane set,
|
||||||
|
runtime override value, reference digest or target identity, and provenance
|
||||||
|
fingerprint;
|
||||||
|
- an omitted explicit input key falls back to the resolved input module;
|
||||||
|
- missing pipeline ID, pipeline digest, input key, and both source/input digests
|
||||||
|
are rejected with stable category context; and
|
||||||
|
- `RelativePath` is deterministic, slash-separated, relative, confined, and
|
||||||
|
includes the normalized pipeline/input and digest-derived hierarchy without
|
||||||
|
asserting incidental private prefix lengths beyond the documented layout.
|
||||||
|
|
||||||
|
Use independent inputs for each mutation so one case cannot pass because a
|
||||||
|
different field also changed.
|
||||||
|
|
||||||
|
### 1.2 All-stage recorder/loader round trips
|
||||||
|
|
||||||
|
Create `internal/framework/checkpoint/filesystem_test.go`; leave the existing
|
||||||
|
schema-identifier test in `recorder_test.go`.
|
||||||
|
|
||||||
|
Using one real root and one identity, exercise the exported recorder and loader
|
||||||
|
for:
|
||||||
|
|
||||||
|
- source success with a valid document, self-references, metadata, and digest;
|
||||||
|
- extract success with at least one serialized artifact, schema identity,
|
||||||
|
content metadata, chunk provenance, warnings, dependency fingerprints, and
|
||||||
|
one rejected output so `StatusSucceededWithRejections` is round-tripped;
|
||||||
|
- merge success with one serialized artifact, warnings, and dependencies; and
|
||||||
|
- normalize success with one serialized artifact, warnings, and dependencies.
|
||||||
|
|
||||||
|
For each stage, assert the loader returns `Reused`, restores the meaningful
|
||||||
|
values, and preserves serialized bytes and codec identity. Mutate the original
|
||||||
|
inputs after recording and mutate one loaded result before reloading; neither
|
||||||
|
mutation may alter persisted or subsequently loaded state.
|
||||||
|
|
||||||
|
Inspect representative created directories and files to retain the `0700`/
|
||||||
|
`0600` permission contract on platforms where Unix permission bits are
|
||||||
|
meaningful. Do not snapshot the full directory tree or complete JSON documents.
|
||||||
|
|
||||||
|
### 1.3 Invalid and incompatible checkpoint state
|
||||||
|
|
||||||
|
Seed valid state through the recorder, then copy or edit one artifact per
|
||||||
|
subtest. Drive every case through the exported loader method for that stage.
|
||||||
|
Require a non-reused decision and a short category fragment for:
|
||||||
|
|
||||||
|
- missing artifact and malformed JSON;
|
||||||
|
- `WorkspaceSchemaVersionV1` and an unknown workspace schema version;
|
||||||
|
- mismatched checkpoint identity digest;
|
||||||
|
- wrong stage, lane, module, terminal status, or dependency fingerprint;
|
||||||
|
- incomplete serialized artifact kind/schema/schema digest;
|
||||||
|
- malformed base64 and content-digest mismatch;
|
||||||
|
- source document validation or source/output digest mismatch; and
|
||||||
|
- extract, merge, and normalize output-digest mismatch.
|
||||||
|
|
||||||
|
Include one successful `StatusSucceededWithRejections` extract case and prove
|
||||||
|
that non-reusable running, failed, pending, or invalidated statuses remain
|
||||||
|
non-reused. Assert categories, not complete sentences. Missing state should be
|
||||||
|
a normal non-reuse decision; corrupt or incompatible state must never panic or
|
||||||
|
silently reuse.
|
||||||
|
|
||||||
|
### 1.4 Stage validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./internal/framework/checkpoint ./internal/framework/pipeline ./internal/cli
|
||||||
|
go test -race ./internal/framework/checkpoint ./internal/framework/pipeline ./internal/cli
|
||||||
|
```
|
||||||
|
|
||||||
|
Stage 1 is complete when identity selection, every persisted stage, mutation
|
||||||
|
ownership, compatibility rejection, and corrupt-state recovery are protected at
|
||||||
|
the real filesystem boundary without production changes.
|
||||||
|
|
||||||
|
## Stage 2 - Remove fake-only, obsolete, and misleading tests
|
||||||
|
|
||||||
|
### 2.1 Framework contracts and helpers
|
||||||
|
|
||||||
|
In `internal/framework/contracts/contracts_test.go`, delete exactly:
|
||||||
|
|
||||||
|
- `TestFakeExtractorReturnsTypedOutput`;
|
||||||
|
- `TestFakeChunkerReturnsSourcePlan`;
|
||||||
|
- `TestFakeChunkerReceivesPerRunContext`;
|
||||||
|
- `TestFakeExtractorReceivesChunkAndAmbientContext`;
|
||||||
|
- `TestFakeMergeNormalizeAndOutputContracts`; and
|
||||||
|
- `TestReferenceSetDataTypes`.
|
||||||
|
|
||||||
|
After removing those tests, delete fake methods/types or imports only when they
|
||||||
|
have no remaining test use. Retain the compile-time interface assertions and
|
||||||
|
the reference/material/artifact clone and JSON non-leakage tests.
|
||||||
|
|
||||||
|
Delete `TestHelpersReturnValidationResults` from
|
||||||
|
`internal/framework/validate/validate_test.go`. Retain `TestApproved` and
|
||||||
|
`TestRejectedTrimsReasonAndMessage`.
|
||||||
|
|
||||||
|
Delete `internal/modules/generic/normalize/noop/typed_test.go`. Do not remove the
|
||||||
|
no-op normalizer's resolver, registration, runner, production-composition, or
|
||||||
|
maintained-example coverage. Retain the direct append-order merger ordering
|
||||||
|
test.
|
||||||
|
|
||||||
|
### 2.2 Obsolete Scriptorium grounding
|
||||||
|
|
||||||
|
Delete `internal/framework/llm/scriptorium_api_test.go` in full. Do not move its
|
||||||
|
unused API inventory elsewhere. Retain and run `scriptorium_client_test.go`,
|
||||||
|
`asset_registry_test.go`, module-local prompt preparation, cancellation,
|
||||||
|
validation, profile, and credential-redaction tests.
|
||||||
|
|
||||||
|
### 2.3 Misleading D&D integration tests
|
||||||
|
|
||||||
|
In `internal/modules/integration/dnd_spells_runner_test.go`, delete exactly:
|
||||||
|
|
||||||
|
- `TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference`;
|
||||||
|
- `TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput`; and
|
||||||
|
- `TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary`.
|
||||||
|
|
||||||
|
Remove helpers/imports only if unused afterward. Retain
|
||||||
|
`TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor` and
|
||||||
|
`TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt` as the two
|
||||||
|
representative cross-family workflows.
|
||||||
|
|
||||||
|
### 2.4 Stage validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/validate
|
||||||
|
go test ./internal/modules/generic/normalize/noop ./internal/modules/integration ./internal/modules/dnd/...
|
||||||
|
```
|
||||||
|
|
||||||
|
Stage 2 is complete when all named low-value tests are gone, retained owners
|
||||||
|
still pass, and no production file changed.
|
||||||
|
|
||||||
|
## Stage 3 - Consolidate configuration, CLI state, and resolution ownership
|
||||||
|
|
||||||
|
### 3.1 Version 3 configuration tests
|
||||||
|
|
||||||
|
Before deleting `internal/core/config/v3_test.go`, preserve its only distinct
|
||||||
|
protections:
|
||||||
|
|
||||||
|
- add a version-2 input to the strict file-decoding cases in
|
||||||
|
`file_config_contract_test.go`; require rejection and the migration category,
|
||||||
|
not the complete diagnostic; and
|
||||||
|
- ensure `env_contract_test.go` positively proves that empty configured cache
|
||||||
|
roots resolve to distinct `notarius/chunk-plans` and
|
||||||
|
`notarius/checkpoints` descendants of the supplied per-user cache root.
|
||||||
|
|
||||||
|
Then delete `v3_test.go`. Do not duplicate its defaults, precedence,
|
||||||
|
redaction, invalid-source, or removed-field cases elsewhere.
|
||||||
|
|
||||||
|
### 3.2 CLI state and maintained examples
|
||||||
|
|
||||||
|
Move `emptyLookup` from `internal/cli/state_surfaces_test.go` to
|
||||||
|
`internal/cli/contract_test_helpers_test.go`, then delete
|
||||||
|
`state_surfaces_test.go` in full. Confirm its remaining behavior is still owned
|
||||||
|
by command/run contracts and `state_hardening_test.go`; do not transplant its
|
||||||
|
tests.
|
||||||
|
|
||||||
|
In `internal/cli/production_contract_test.go`:
|
||||||
|
|
||||||
|
- keep `TestDefaultCLICompositionResolvesMaintainedConfigurations`, but rename
|
||||||
|
it to `TestDefaultCLICompositionValidatesRepresentativeConfiguration` and
|
||||||
|
reduce it to one representative `config validate` command using empty/default
|
||||||
|
`Options`; this test owns fallback production composition, not
|
||||||
|
maintained-example enumeration;
|
||||||
|
- remove maintained-example resolution from
|
||||||
|
`TestProductionCatalogCoversMaintainedConfigurations` while retaining
|
||||||
|
required production registry members, typed codec/variant wiring, catalog
|
||||||
|
conversion, and the exact documented spell validator chain; and
|
||||||
|
- remove the maintained-example loop from
|
||||||
|
`TestProductionConfigValidationCoversModuleAndVariantFailures`, retaining one
|
||||||
|
valid baseline and each distinct failure mutation.
|
||||||
|
|
||||||
|
Do not weaken `example_contract_test.go`; it remains the sole owner for loading,
|
||||||
|
resolving, listing, and executing the maintained examples.
|
||||||
|
|
||||||
|
### 3.3 Default and Seriatim resolver duplication
|
||||||
|
|
||||||
|
Delete `internal/framework/pipeline/default_modules_test.go` in full.
|
||||||
|
|
||||||
|
Delete `internal/modules/seriatim/input/transcript/config_test.go` in full.
|
||||||
|
Delete `internal/modules/seriatim/input/transcript/testdata/pipeline.yml` with
|
||||||
|
it; the fixture is owned only by that deleted test file. Retain adapter parsing,
|
||||||
|
leaf registration, the Seriatim runner integration, generic resolver tests,
|
||||||
|
and production examples.
|
||||||
|
|
||||||
|
### 3.4 D&D capability integration
|
||||||
|
|
||||||
|
Rewrite `internal/modules/integration/dnd_spells_config_test.go` to own only two
|
||||||
|
cross-family capability failures:
|
||||||
|
|
||||||
|
1. removing `source.transcript` from the Seriatim input spec must make the D&D
|
||||||
|
spell extractor incompatible; and
|
||||||
|
2. removing `dnd.spell_casts` from the spell extractor spec must make the
|
||||||
|
append-order merger incompatible.
|
||||||
|
|
||||||
|
Use one compact table over a programmatically constructed profile and the
|
||||||
|
smallest catalog capable of resolution. Delete the successful-resolution,
|
||||||
|
stable-digest, and unknown-lane cases. Simplify or remove fixture/catalog
|
||||||
|
helpers that become unnecessary, but preserve helpers used by the retained
|
||||||
|
runner integration in sibling test files.
|
||||||
|
|
||||||
|
### 3.5 Stage validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./internal/core/config ./internal/cli ./internal/framework/pipeline
|
||||||
|
go test ./internal/modules/seriatim/... ./internal/modules/integration
|
||||||
|
```
|
||||||
|
|
||||||
|
Stage 3 is complete when each behavior has the owner specified above, the
|
||||||
|
maintained examples are not redundantly enumerated, and all distinct migration,
|
||||||
|
default-composition, and capability protections remain.
|
||||||
|
|
||||||
|
## Stage 4 - Rewrite brittle assertions at durable behavioral boundaries
|
||||||
|
|
||||||
|
### 4.1 Source validation ownership and diagnostics
|
||||||
|
|
||||||
|
In `internal/core/source/source_test.go`:
|
||||||
|
|
||||||
|
- consolidate malformed `SourceRef` categories under the `ValidateRef` tests;
|
||||||
|
- reduce `TestValidateDocumentUnitReferences` to one case proving nested
|
||||||
|
reference failures receive unit/document context and one case proving the
|
||||||
|
document-only self-reference invariant;
|
||||||
|
- retain valid documents, required document/unit fields, duplicate IDs,
|
||||||
|
non-empty units, valid/reversed/missing references, unit lookup, and digest
|
||||||
|
sensitivity/determinism; and
|
||||||
|
- replace exact complete error equality with the minimum stable field/category
|
||||||
|
fragments needed to distinguish each failure.
|
||||||
|
|
||||||
|
Do not add typed production errors during this pass. If stable fragments cannot
|
||||||
|
distinguish meaningful categories without a production change, retain the
|
||||||
|
narrowest current assertion and report that limitation rather than changing
|
||||||
|
production code.
|
||||||
|
|
||||||
|
### 4.2 Private LLM response schemas
|
||||||
|
|
||||||
|
In `internal/modules/dnd/chunk/scenes/schema_test.go`, replace the nested
|
||||||
|
`map[string]any` schema-structure walk with actual JSON Schema validation.
|
||||||
|
Add a small test helper that parses the instance and schema with
|
||||||
|
`jsonschema.UnmarshalJSON`, registers the schema with
|
||||||
|
`jsonschema.NewCompiler().AddResource`, compiles it, and calls
|
||||||
|
`schema.Validate`, matching the existing production validator boundary. Use a
|
||||||
|
representative valid scene response and mutations that reject:
|
||||||
|
|
||||||
|
- obsolete segment-based boundary fields;
|
||||||
|
- non-positive start/end unit IDs;
|
||||||
|
- invalid `primary_mode` and `boundary_confidence` values;
|
||||||
|
- empty boundary caveats; and
|
||||||
|
- unknown properties.
|
||||||
|
|
||||||
|
Retain identity/hash validity, DTO integer decoding, and mutation safety.
|
||||||
|
|
||||||
|
In `internal/modules/dnd/extract/spells/schema_test.go`, validate one legal
|
||||||
|
private LLM response whose references omit `source_id`, and prove that adding a
|
||||||
|
canonical `source_id` is rejected. Retain response identity/hash, mutation
|
||||||
|
safety, and diagnostics non-leakage. Do not conflate this private transport
|
||||||
|
schema with the durable codec schema or alter the maintained durable fixture.
|
||||||
|
|
||||||
|
### 4.3 Framework schema enumeration
|
||||||
|
|
||||||
|
In `internal/framework/llm/schema_registry_test.go`, remove only the assertion
|
||||||
|
that `RegisteredResponseSchemas` has exactly two entries. Continue to assert:
|
||||||
|
|
||||||
|
- returned keys are sorted;
|
||||||
|
- both required framework test schemas are present and valid;
|
||||||
|
- returned bytes are mutation-safe;
|
||||||
|
- diagnostics omit raw schema content; and
|
||||||
|
- D&D schemas are not registered in the domain-neutral framework registry.
|
||||||
|
|
||||||
|
### 4.4 Stage validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./internal/core/source ./internal/framework/llm
|
||||||
|
go test ./internal/modules/dnd/chunk/scenes ./internal/modules/dnd/extract/spells ./internal/modules/dnd/codec/spells
|
||||||
|
```
|
||||||
|
|
||||||
|
Stage 4 is complete when failures express behavioral categories, private
|
||||||
|
schemas are validated by accepted/rejected instances, and durable schema
|
||||||
|
compatibility coverage remains unchanged.
|
||||||
|
|
||||||
|
## Stage 5 - Make composition and architecture checks extension-friendly
|
||||||
|
|
||||||
|
### 5.1 Family registrars
|
||||||
|
|
||||||
|
Rewrite the success assertions in:
|
||||||
|
|
||||||
|
- `internal/modules/generic/register/register_test.go`;
|
||||||
|
- `internal/modules/seriatim/register/register_test.go`; and
|
||||||
|
- `internal/modules/dnd/register/register_test.go`.
|
||||||
|
|
||||||
|
Use required-membership helpers rather than exact equality for registered keys
|
||||||
|
and asset names. Prove representative family-owned entries are retrievable or
|
||||||
|
buildable through their registry boundary. Preserve:
|
||||||
|
|
||||||
|
- validation of all required registry/asset dependencies before any mutation;
|
||||||
|
- contextual failure on duplicate family registration;
|
||||||
|
- absence of cross-family composition where that is an explicit architectural
|
||||||
|
ownership rule; and
|
||||||
|
- exact D&D spell default-validator order, because it is documented production
|
||||||
|
policy.
|
||||||
|
|
||||||
|
Do not delete leaf spec/constructor/registration tests and do not move the CLI
|
||||||
|
production catalog into family tests.
|
||||||
|
|
||||||
|
### 5.2 Central import-boundary enforcement
|
||||||
|
|
||||||
|
Move the two rules from
|
||||||
|
`internal/framework/chunkplan/import_boundaries_test.go` into
|
||||||
|
`internal/modules/import_boundaries_test.go`:
|
||||||
|
|
||||||
|
- production files under `internal/core/source` may import neither
|
||||||
|
`internal/framework` nor `internal/modules`; and
|
||||||
|
- production files under `internal/framework/chunkplan` may not import
|
||||||
|
`internal/modules`.
|
||||||
|
|
||||||
|
Extend the centralized checker so its repository walk enforces those rules.
|
||||||
|
Add rule-level synthetic cases for both allowed and forbidden imports, using
|
||||||
|
the existing table/checker style. Retain the existing rejected fixture that
|
||||||
|
proves generic-to-concrete enforcement. Once both unique rules and checker
|
||||||
|
activation are protected centrally, delete
|
||||||
|
`internal/framework/chunkplan/import_boundaries_test.go`.
|
||||||
|
|
||||||
|
### 5.3 Internal documentation routing
|
||||||
|
|
||||||
|
Update only the `Tests To Inspect` routing needed to match the final suite in:
|
||||||
|
|
||||||
|
- `docs/internal/state.md`;
|
||||||
|
- `docs/internal/pipeline.md`;
|
||||||
|
- `docs/internal/modules.md`; and
|
||||||
|
- `docs/internal/llm.md`.
|
||||||
|
|
||||||
|
Replace deleted or nonexistent names such as
|
||||||
|
`internal/cli/state_surfaces_test.go` and `internal/cli/run_test.go` with concise
|
||||||
|
links or paths to the retained command/run contracts, state-hardening,
|
||||||
|
production-composition, maintained-example, checkpoint filesystem, and
|
||||||
|
cross-family integration owners. Do not create an exhaustive test inventory or
|
||||||
|
repeat subsystem contracts owned elsewhere.
|
||||||
|
|
||||||
|
### 5.4 Stage validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./internal/modules/... ./internal/framework/chunkplan
|
||||||
|
go test ./internal/cli ./internal/framework/pipeline ./internal/framework/llm
|
||||||
|
```
|
||||||
|
|
||||||
|
Stage 5 is complete when legitimate family additions no longer require closed
|
||||||
|
inventory edits, all architectural rules remain executable in one checker, and
|
||||||
|
internal documentation names only existing retained owners.
|
||||||
|
|
||||||
|
## Stage 6 - Repository acceptance and handoff
|
||||||
|
|
||||||
|
1. Review the final diff against every required finding and retained-owner
|
||||||
|
statement in [Test Suite Policy Review](tests.md). Confirm that no production
|
||||||
|
`.go` file changed.
|
||||||
|
2. Run formatting on changed Go test files, then run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./cmd/notarius
|
||||||
|
go test -race ./internal/framework/pipeline ./internal/framework/llm ./internal/framework/checkpoint ./internal/cli ./internal/modules/integration
|
||||||
|
go test ./... -shuffle=on -count=5
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Generate one coverage report for investigation. Confirm the new checkpoint
|
||||||
|
tests execute source, extract, merge, normalize, manifest-validation, and
|
||||||
|
corruption paths. Do not compare the percentage to the review baseline and
|
||||||
|
do not add tests merely to increase it.
|
||||||
|
4. Check documentation links and `git diff --check`. Confirm no fixtures were
|
||||||
|
updated automatically and no secrets or external-service requirements were
|
||||||
|
introduced.
|
||||||
|
|
||||||
|
Implementation is complete when:
|
||||||
|
|
||||||
|
- every checkpoint stage has a real filesystem round trip and corrupted or
|
||||||
|
incompatible state is never silently reused;
|
||||||
|
- every deletion or rewrite in the feature roadmap is complete while its named
|
||||||
|
retained owner still passes;
|
||||||
|
- maintained examples have one example-contract owner plus one narrow default
|
||||||
|
production-composition smoke path;
|
||||||
|
- registrar and architecture checks permit legitimate extension without
|
||||||
|
weakening documented ownership or validator order;
|
||||||
|
- the full suite is deterministic, offline, credential-free, race-clean, and
|
||||||
|
diagnostically useful;
|
||||||
|
- internal documentation points to existing test owners; and
|
||||||
|
- all validation commands pass with no production behavior change.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None. The feature roadmap contains enough evidence and policy decisions to
|
||||||
|
implement every stage without additional testing-policy choices. If a new test
|
||||||
|
reveals a production defect, that is a scope boundary rather than an open
|
||||||
|
planning question: stop the affected stage and request explicit authorization
|
||||||
|
before changing production behavior.
|
||||||
@@ -23,8 +23,9 @@ important behavior.
|
|||||||
## Objective
|
## Objective
|
||||||
|
|
||||||
Review the complete Notarius test suite against the Testing Policy, identify
|
Review the complete Notarius test suite against the Testing Policy, identify
|
||||||
both low-value coverage and meaningful protection gaps, and then develop a
|
both low-value coverage and meaningful protection gaps, and define the
|
||||||
decision-complete staged implementation plan for the justified changes.
|
justified target revisions. The ordered execution plan is maintained separately
|
||||||
|
in [Implementation](implementation.md).
|
||||||
|
|
||||||
The review must answer:
|
The review must answer:
|
||||||
|
|
||||||
@@ -151,27 +152,6 @@ group shares one clear disposition.
|
|||||||
Distinguish required changes from optional cleanup. Absence of a finding is not
|
Distinguish required changes from optional cleanup. Absence of a finding is not
|
||||||
evidence that a package needs more tests.
|
evidence that a package needs more tests.
|
||||||
|
|
||||||
## Implementation-Plan Deliverable
|
|
||||||
|
|
||||||
After completing the review, write a decision-complete staged implementation
|
|
||||||
plan based on the supported findings. The plan must:
|
|
||||||
|
|
||||||
- order work in small, independently verifiable package or behavior groups;
|
|
||||||
- state exactly which protections are retained when tests are consolidated,
|
|
||||||
rewritten, or deleted;
|
|
||||||
- add tests only for identified meaningful risks;
|
|
||||||
- avoid production behavior changes unless a separately identified production
|
|
||||||
defect is expressly brought into scope;
|
|
||||||
- identify focused and repository-wide validation for each stage; and
|
|
||||||
- include acceptance criteria demonstrating that the resulting suite remains
|
|
||||||
deterministic, offline, diagnostically useful, and sufficient under the
|
|
||||||
Testing Policy.
|
|
||||||
|
|
||||||
The plan must not establish a coverage-percentage target, require mechanical
|
|
||||||
conversion to table-driven tests, restore old tests wholesale, or equate fewer
|
|
||||||
tests with success. It should prefer correct, idiomatic, and maintainable tests
|
|
||||||
even when achieving the durable boundary requires more immediate work.
|
|
||||||
|
|
||||||
## Completion Criteria
|
## Completion Criteria
|
||||||
|
|
||||||
The review is complete when:
|
The review is complete when:
|
||||||
@@ -183,5 +163,299 @@ The review is complete when:
|
|||||||
- redundant or brittle protection is distinguished from valuable intentional
|
- redundant or brittle protection is distinguished from valuable intentional
|
||||||
overlap;
|
overlap;
|
||||||
- production defects, if any, are reported separately; and
|
- production defects, if any, are reported separately; and
|
||||||
- the resulting implementation plan can be executed without requiring the
|
- every target revision is specific enough to support a decision-complete
|
||||||
implementing agent to make additional testing-policy decisions.
|
implementation roadmap.
|
||||||
|
|
||||||
|
## Revisions Needed
|
||||||
|
|
||||||
|
### Review baseline
|
||||||
|
|
||||||
|
The review was completed on 2026-07-18 against 90 committed Go test files,
|
||||||
|
536 named tests, approximately 21,900 lines of test code, and the committed
|
||||||
|
fixtures under `testdata/` and `examples/`.
|
||||||
|
|
||||||
|
The baseline is clean:
|
||||||
|
|
||||||
|
- `go test ./...`, `go vet ./...`, and `go build ./cmd/notarius` pass;
|
||||||
|
- `go test -race` passes for `internal/framework/pipeline`,
|
||||||
|
`internal/framework/llm`, `internal/cli`, and
|
||||||
|
`internal/modules/integration`;
|
||||||
|
- five shuffled repetitions of the full suite pass; and
|
||||||
|
- the default suite is offline, credential-free, and fast. The only HTTP
|
||||||
|
behavior uses local test servers or deliberately unreachable loopback
|
||||||
|
endpoints without making provider calls.
|
||||||
|
|
||||||
|
Coverage was used only to investigate risk. It confirms broad behavioral
|
||||||
|
coverage in CLI, configuration, source, pipeline, and production module
|
||||||
|
packages. It also exposes the material checkpoint gap described below: most
|
||||||
|
filesystem loader paths and all source, merge, and normalize checkpoint
|
||||||
|
round trips are unexercised. No production defect was identified during this
|
||||||
|
review.
|
||||||
|
|
||||||
|
### Test ownership that should be retained
|
||||||
|
|
||||||
|
The following overlap is intentional and should remain:
|
||||||
|
|
||||||
|
- CLI command, run, cache, reference, state-hardening, production-composition,
|
||||||
|
and maintained-example tests own exit classification, option mapping,
|
||||||
|
physical state placement, default production wiring, and representative
|
||||||
|
assembled workflows. Package tests continue to own the underlying parsing,
|
||||||
|
resolution, and persistence rules.
|
||||||
|
- Configuration contract tests own file decoding, precedence, validation,
|
||||||
|
redaction, and effective resolution. Pipeline resolver tests own module,
|
||||||
|
capability, typed-variant, validator-chain, and reference resolution after
|
||||||
|
configuration has produced a profile.
|
||||||
|
- Pipeline runner tests own bounded concurrency, deterministic ordering,
|
||||||
|
cancellation, retries, rejection propagation, checkpoint decisions, debug
|
||||||
|
recording, candidate/final encoding, and terminal failure behavior. The one
|
||||||
|
cross-family concurrency test remains valuable because it proves that the
|
||||||
|
framework worker bound and shared provider scheduler remain independent in
|
||||||
|
an assembled production-style pipeline.
|
||||||
|
- Source, chunk-plan store, file I/O, debug-bundle, output encoder, artifact
|
||||||
|
codec, and manifest tests own data integrity, confinement, serialization,
|
||||||
|
compatibility, atomic publication, redaction, and round trips. The exact
|
||||||
|
checkpoint schema identifier and exact artifact-schema digest tests are
|
||||||
|
compatibility tests, not incidental constant assertions, and must remain.
|
||||||
|
- Seriatim adapter, D&D scene chunker and spell extractor, D&D validators,
|
||||||
|
generic chunker and validators, prompt preparation, and durable spell codec
|
||||||
|
tests own their domain rules and external or durable schemas. Leaf
|
||||||
|
constructor/spec/registration tests remain justified because the extension
|
||||||
|
contract explicitly requires each leaf to be independently buildable.
|
||||||
|
- The centralized production import-boundary test remains a valuable
|
||||||
|
executable architecture rule. Its synthetic rejected fixture is necessary
|
||||||
|
to prove that the checker itself is active.
|
||||||
|
|
||||||
|
Absence from the revisions below means the current behavioral group has an
|
||||||
|
appropriate owner and no justified change was found.
|
||||||
|
|
||||||
|
### Required findings
|
||||||
|
|
||||||
|
#### P0 - add filesystem checkpoint compatibility and recovery coverage
|
||||||
|
|
||||||
|
**Disposition: add.** `internal/framework/checkpoint/recorder_test.go` currently
|
||||||
|
proves only that an empty successful extract checkpoint can be written and
|
||||||
|
loaded. Coverage confirms that `FilesystemLoader.Source`, `Merge`, `Normalize`,
|
||||||
|
manifest validation, payload digest validation, and most recorder status paths
|
||||||
|
are otherwise silent. CLI resume coverage proves root selection and one happy
|
||||||
|
reuse, while pipeline checkpoint tests use controlled collaborators; neither
|
||||||
|
owns the filesystem format.
|
||||||
|
|
||||||
|
Add package-level tests at the real recorder/loader boundary that:
|
||||||
|
|
||||||
|
- prove `NewIdentity` is deterministic under reordered lanes, references, and
|
||||||
|
fingerprints, changes when pipeline, source/input, selected lanes, runtime
|
||||||
|
overrides, references, or provenance change, and produces a confined stable
|
||||||
|
relative path;
|
||||||
|
- round-trip representative source, extract, merge, and normalize payloads,
|
||||||
|
including serialized artifact identity, warnings, extract rejections,
|
||||||
|
metadata, and dependency fingerprints;
|
||||||
|
- prove caller mutation cannot alter recorded or loaded values and retain the
|
||||||
|
existing restrictive-permission expectation through real files; and
|
||||||
|
- mutate one persisted artifact at a time to prove that missing or malformed
|
||||||
|
JSON, old or unknown workspace schema versions, wrong identity/stage/lane/
|
||||||
|
module/status/dependencies, incomplete codec identity, invalid base64, and
|
||||||
|
content/output-digest mismatches yield a non-reused decision with useful
|
||||||
|
category context rather than a panic or silent reuse.
|
||||||
|
|
||||||
|
Use relationships and category fragments rather than snapshotting complete
|
||||||
|
manifests or error sentences. This addition protects cache correctness,
|
||||||
|
compatibility, recovery, and sensitive durable state; it is not intended to
|
||||||
|
raise a coverage percentage.
|
||||||
|
|
||||||
|
#### P1 - remove tests that exercise only their own fakes or obsolete APIs
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove the fake-behavior tests in
|
||||||
|
`internal/framework/contracts/contracts_test.go`:
|
||||||
|
`TestFakeExtractorReturnsTypedOutput`, `TestFakeChunkerReturnsSourcePlan`,
|
||||||
|
`TestFakeChunkerReceivesPerRunContext`,
|
||||||
|
`TestFakeExtractorReceivesChunkAndAmbientContext`,
|
||||||
|
`TestFakeMergeNormalizeAndOutputContracts`, and `TestReferenceSetDataTypes`.
|
||||||
|
They detect changes in test helpers or Go struct assignment, not defects in
|
||||||
|
production contracts. Retain the compile-time interface assertions and the
|
||||||
|
clone, JSON omission, and content-ownership tests, which protect type
|
||||||
|
compatibility and non-leakage invariants.
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove
|
||||||
|
`internal/framework/llm/scriptorium_api_test.go`. It was introduced as
|
||||||
|
pre-adapter API grounding and still mirrors unused Scriptorium request/result
|
||||||
|
fields, option constructors, and sentinel errors. Production compilation now
|
||||||
|
grounds the API actually used, while `scriptorium_client_test.go` and module
|
||||||
|
prompt-preparation tests protect adapter mapping, local integration,
|
||||||
|
cancellation, validation, profile selection, and credential redaction.
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove
|
||||||
|
`TestHelpersReturnValidationResults` from
|
||||||
|
`internal/framework/validate/validate_test.go`; it is a compile-only assignment
|
||||||
|
that adds no protection beyond the function signatures and the two behavioral
|
||||||
|
helper tests.
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove the direct
|
||||||
|
`TestTypedNormalizerPreservesValue` test for the generic no-op normalizer. Its
|
||||||
|
single assignment is already exercised through resolver, runner, production
|
||||||
|
composition, and maintained-example output tests. Retain the direct append-order
|
||||||
|
merger test because input ordering is a meaningful transformation invariant.
|
||||||
|
|
||||||
|
#### P1 - remove misleading D&D integration claims
|
||||||
|
|
||||||
|
**Disposition: delete.** In
|
||||||
|
`internal/modules/integration/dnd_spells_runner_test.go`, delete:
|
||||||
|
|
||||||
|
- `TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference`, whose fake LLM
|
||||||
|
is programmed to return no spells and therefore cannot prove the claimed
|
||||||
|
extraction policy;
|
||||||
|
- `TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput`, which
|
||||||
|
omits the production validator chain and duplicates the extractor's explicit
|
||||||
|
invalid-evidence handoff test; and
|
||||||
|
- `TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary`, whose
|
||||||
|
final-codec invariant is already owned by the codec and runner candidate/
|
||||||
|
final-encoding tests.
|
||||||
|
|
||||||
|
Retain the representative Seriatim-to-spell assembled workflow and the test
|
||||||
|
that proves party/glossary references and provenance cross the production
|
||||||
|
module boundary. The policy that reference material is not source evidence is
|
||||||
|
durably protected by extractor mapping, prompt-input assertions, source-ref
|
||||||
|
validators, and the production validator-chain composition test; it cannot be
|
||||||
|
proved by controlling an LLM stub's semantic answer.
|
||||||
|
|
||||||
|
#### P1 - consolidate superseded configuration and CLI state tests
|
||||||
|
|
||||||
|
**Disposition: consolidate.** Delete `internal/core/config/v3_test.go` after
|
||||||
|
moving its two distinct protections into the current contract owners:
|
||||||
|
|
||||||
|
- add the version-2 migration rejection case to the strict file-decoding cases
|
||||||
|
in `file_config_contract_test.go`; and
|
||||||
|
- ensure the positive per-user chunk-plan/checkpoint root separation remains
|
||||||
|
in `env_contract_test.go`.
|
||||||
|
|
||||||
|
The remaining default, precedence, invalid-source, redaction, and removed-field
|
||||||
|
assertions are already more completely owned by `file_config_contract_test.go`,
|
||||||
|
`env_contract_test.go`, `validation_contract_test.go`, and
|
||||||
|
`redaction_test.go`.
|
||||||
|
|
||||||
|
**Disposition: consolidate.** Delete `internal/cli/state_surfaces_test.go`.
|
||||||
|
Move the shared `emptyLookup` helper to
|
||||||
|
`contract_test_helpers_test.go`. Its debug flag syntax, allocation timing,
|
||||||
|
no-debug absence, version-3 validation, and removed-field cases are already
|
||||||
|
owned by `run_contract_test.go`, `state_hardening_test.go`,
|
||||||
|
`command_contract_test.go`, and the configuration contract tests. Preserve the
|
||||||
|
state matrix, failure retention, no-debug terminal-writer, and pre-resolution
|
||||||
|
debug allocation protections in those current owners.
|
||||||
|
|
||||||
|
**Disposition: rewrite.** Keep
|
||||||
|
`TestDefaultCLICompositionResolvesMaintainedConfigurations` as a narrow default
|
||||||
|
composition smoke test, but run one representative config-validation command
|
||||||
|
through empty `Options` rather than revalidating every maintained example.
|
||||||
|
`example_contract_test.go` owns both maintained examples. Remove the repeated
|
||||||
|
maintained-example loops from `TestProductionCatalogCoversMaintainedConfigurations`
|
||||||
|
and `TestProductionConfigValidationCoversModuleAndVariantFailures`; retain the
|
||||||
|
production registry/codec/default-chain checks and the distinct production
|
||||||
|
failure cases.
|
||||||
|
|
||||||
|
#### P1 - consolidate resolver and module-composition duplication
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove
|
||||||
|
`internal/framework/pipeline/default_modules_test.go`. Its large local catalog
|
||||||
|
only re-proves default binding selection, already owned by
|
||||||
|
`profile_test.go` and `effective_config_contract_test.go`; production default
|
||||||
|
keys and wiring are separately exercised by leaf registration, the production
|
||||||
|
catalog, default CLI composition, and the maintained end-to-end example.
|
||||||
|
|
||||||
|
**Disposition: delete.** Remove
|
||||||
|
`internal/modules/seriatim/input/transcript/config_test.go`. It builds an
|
||||||
|
entire fake catalog to repeat generic resolver success, deterministic digest,
|
||||||
|
missing-capability, and unknown-lane behavior. Retain Seriatim parsing and leaf
|
||||||
|
registration tests, the Seriatim runner integration, D&D cross-family
|
||||||
|
capability tests, resolver contract tests, and production CLI examples.
|
||||||
|
|
||||||
|
**Disposition: consolidate.** In
|
||||||
|
`internal/modules/integration/dnd_spells_config_test.go`, retain only the two
|
||||||
|
cross-family capability contracts: Seriatim must provide `source.transcript`
|
||||||
|
to the spell extractor, and the spell extractor must provide
|
||||||
|
`dnd.spell_casts` to append-order. Express them as one compact table over a
|
||||||
|
programmatic profile/catalog. Remove the generic successful-resolution,
|
||||||
|
stable-digest, and unknown-lane cases, which are already exercised by the
|
||||||
|
retained runner integration and resolver/configuration owners.
|
||||||
|
|
||||||
|
#### P1 - replace brittle assertions with behavioral ones
|
||||||
|
|
||||||
|
**Disposition: rewrite.** In `internal/core/source/source_test.go`, make
|
||||||
|
`ValidateRef` the owner of malformed reference categories. `ValidateDocument`
|
||||||
|
should retain one propagation/context case plus the document-only self-reference
|
||||||
|
invariant instead of repeating missing, foreign, and reversed-reference cases.
|
||||||
|
Replace complete internal error-sentence equality with the smallest stable
|
||||||
|
category and field fragments. Retain exact acceptance/rejection boundaries,
|
||||||
|
duplicate detection, deterministic digests, and reference ordering.
|
||||||
|
|
||||||
|
**Disposition: rewrite.** Replace the decoded-schema-structure walk in
|
||||||
|
`internal/modules/dnd/chunk/scenes/schema_test.go` with representative JSON
|
||||||
|
Schema validation: accept a valid source-unit-boundary response and reject old
|
||||||
|
segment fields, non-positive bounds, invalid enums, empty caveats, and unknown
|
||||||
|
properties. Likewise, change the private spell response-schema assertion in
|
||||||
|
`internal/modules/dnd/extract/spells/schema_test.go` to accept the LLM transport
|
||||||
|
shape without `source_id` and reject a response that supplies canonical
|
||||||
|
`source_id`. Retain schema identity/hash, mutation safety, diagnostics
|
||||||
|
non-leakage, DTO decoding, and the codec's durable fixture tests. These rewrites
|
||||||
|
preserve schema regression protection without coupling tests to nested map
|
||||||
|
layout or unsafe type assertions.
|
||||||
|
|
||||||
|
**Disposition: rewrite.** In `internal/framework/llm/schema_registry_test.go`,
|
||||||
|
assert that returned keys are sorted and required framework test schemas are
|
||||||
|
present without asserting that the registry contains exactly two entries.
|
||||||
|
Retain the explicit negative D&D lookup because it protects the framework/
|
||||||
|
domain ownership boundary.
|
||||||
|
|
||||||
|
#### P2 - make composition and architecture tests open to legitimate extension
|
||||||
|
|
||||||
|
**Disposition: rewrite.** The family registrar tests in
|
||||||
|
`internal/modules/generic/register`, `internal/modules/seriatim/register`, and
|
||||||
|
`internal/modules/dnd/register` currently assert closed-world key and asset
|
||||||
|
inventories that duplicate the CLI production catalog. Change them to require
|
||||||
|
the family-owned registrations they need, permit unrelated future additions,
|
||||||
|
and prove representative entries can be built or prepared. Keep exact order
|
||||||
|
for the D&D default spell validator chain because that order is documented
|
||||||
|
production policy. Keep missing-dependency-before-mutation and contextual
|
||||||
|
duplicate-registration cases.
|
||||||
|
|
||||||
|
**Disposition: consolidate.** Move the source/framework independence rules
|
||||||
|
from `internal/framework/chunkplan/import_boundaries_test.go` into the
|
||||||
|
centralized `internal/modules/import_boundaries_test.go` checker and delete the
|
||||||
|
second repository walker. Preserve both unique rules: `internal/core/source`
|
||||||
|
may import neither framework nor modules, and `internal/framework/chunkplan`
|
||||||
|
may not import modules. Add rule-level cases so a broken checker still fails.
|
||||||
|
|
||||||
|
#### Documentation alignment
|
||||||
|
|
||||||
|
**Disposition: rewrite.** When the affected tests move or are deleted, update
|
||||||
|
the `Tests To Inspect` sections in `docs/internal/state.md`,
|
||||||
|
`docs/internal/pipeline.md`, `docs/internal/modules.md`, and
|
||||||
|
`docs/internal/llm.md`. They currently name files such as
|
||||||
|
`internal/cli/state_surfaces_test.go` and `internal/cli/run_test.go` that will
|
||||||
|
be deleted or do not exist. Point each document at the retained contract,
|
||||||
|
state-hardening, production-composition, example, checkpoint, and integration
|
||||||
|
owners without recreating an exhaustive test inventory.
|
||||||
|
|
||||||
|
### Optional cleanup
|
||||||
|
|
||||||
|
No additional cleanup is recommended now. In particular, do not mechanically
|
||||||
|
convert the large resolver and runner suites to table-driven form, merge all
|
||||||
|
leaf registration tests into family registrars, add tests for trivial accessor
|
||||||
|
coverage, or introduce a golden-update framework. Those changes do not provide
|
||||||
|
enough additional confidence to justify their immediate cost.
|
||||||
|
|
||||||
|
### Target state
|
||||||
|
|
||||||
|
The revision is complete when:
|
||||||
|
|
||||||
|
- every checkpoint stage has a real filesystem round trip and incompatible or
|
||||||
|
corrupted state is demonstrably recomputed rather than silently reused;
|
||||||
|
- all protection named as retained above remains present at its stated owner;
|
||||||
|
- the fake-only, obsolete, misleading, duplicate, and closed-world assertions
|
||||||
|
named in the required findings are removed or rewritten exactly as specified;
|
||||||
|
- maintained examples are each owned by one example contract plus one narrow
|
||||||
|
default-composition smoke path, rather than repeated across production tests;
|
||||||
|
- default tests remain deterministic, offline, credential-free, order
|
||||||
|
independent, and race-clean;
|
||||||
|
- failures identify the violated behavioral category without snapshotting
|
||||||
|
complete incidental diagnostics;
|
||||||
|
- internal documentation points to existing retained test owners; and
|
||||||
|
- the suite reaches this state without changing production behavior.
|
||||||
|
|||||||
@@ -12,3 +12,5 @@ func replaceRequiredOnce(t *testing.T, input, old, replacement string) string {
|
|||||||
}
|
}
|
||||||
return strings.Replace(input, old, replacement, 1)
|
return strings.Replace(input, old, replacement, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func emptyLookup(string) (string, bool) { return "", false }
|
||||||
|
|||||||
@@ -91,40 +91,15 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
|
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
|
||||||
cfg := loadMaintainedExample(t, example.path)
|
|
||||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolve %s: %v", example.name, err)
|
|
||||||
}
|
|
||||||
if effective.ResolvedPipeline.Input.Module != "seriatim" || len(effective.ResolvedPipeline.ArtifactLanes) != 1 || effective.ResolvedPipeline.ArtifactLanes[0].ID != "spells" {
|
|
||||||
t.Fatalf("resolved %s pipeline = %#v, want seriatim and spells", example.name, effective.ResolvedPipeline)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDefaultCLICompositionResolvesMaintainedConfigurations(t *testing.T) {
|
func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
|
||||||
catalog, err := effectiveCatalog(Options{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolve default catalog: %v", err)
|
|
||||||
}
|
|
||||||
if isEmptyCatalog(catalog) {
|
|
||||||
t.Fatal("default catalog is empty")
|
|
||||||
}
|
|
||||||
registries, err := effectiveRegistries(Options{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolve default registries: %v", err)
|
|
||||||
}
|
|
||||||
if isEmptyRegistries(registries) {
|
|
||||||
t.Fatal("default registries are empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
|
||||||
var stdout, stderr strings.Builder
|
var stdout, stderr strings.Builder
|
||||||
code := RunWithOptions([]string{"config", "validate", "--config", example.path, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{LookupEnv: emptyLookup})
|
code := RunWithOptions([]string{
|
||||||
|
"config", "validate", "--config", repositoryPath("examples", "dnd-spells.config.yml"), "--pipeline", "dnd-session",
|
||||||
|
}, &stdout, &stderr, Options{})
|
||||||
if code != 0 || stderr.Len() != 0 {
|
if code != 0 || stderr.Len() != 0 {
|
||||||
t.Fatalf("validate maintained %s config with defaults: code=%d stdout=%q stderr=%q", example.name, code, stdout.String(), stderr.String())
|
t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -212,14 +187,6 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
|
|||||||
if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
|
if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
|
||||||
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
}
|
}
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
|
||||||
var exampleStdout, exampleStderr strings.Builder
|
|
||||||
code := RunWithOptions([]string{"config", "validate", "--config", example.path, "--pipeline", "dnd-session"}, &exampleStdout, &exampleStderr, options)
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("validate maintained %s config: code=%d stdout=%q stderr=%q", example.name, code, exampleStdout.String(), exampleStderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
content string
|
content string
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const stateSurfaceRunID = "run-1000000000-55555555555555555555555555555555"
|
|
||||||
|
|
||||||
func stateSurfaceRunIDGenerator(time.Time) (string, error) { return stateSurfaceRunID, nil }
|
|
||||||
|
|
||||||
func TestRunRejectsDebugDirectoryWithoutDebug(t *testing.T) {
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{"run", "example", "--input", "source.json", "--debug-dir", t.TempDir()}, &stdout, &stderr, Options{})
|
|
||||||
if code != 2 || !strings.Contains(stderr.String(), "--debug-dir requires --debug") {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDebugAllocatesBeforePipelineResolution(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
configPath := writeV3Config(t, "")
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", "source.json", "--debug", "--debug-dir", root, "--chunk_cache", "bypass"}, &stdout, &stderr, Options{LookupEnv: emptyLookup, RunIDGenerator: stateSurfaceRunIDGenerator})
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(root)
|
|
||||||
if err != nil || len(entries) != 1 {
|
|
||||||
t.Fatalf("debug bundles: %v, %v", entries, err)
|
|
||||||
}
|
|
||||||
if entries[0].Name() != stateSurfaceRunID {
|
|
||||||
t.Fatalf("debug bundle name = %q, want %q", entries[0].Name(), stateSurfaceRunID)
|
|
||||||
}
|
|
||||||
bundle := filepath.Join(root, stateSurfaceRunID)
|
|
||||||
for _, name := range []string{"summary", "trace"} {
|
|
||||||
if info, err := os.Stat(filepath.Join(bundle, name)); err != nil || !info.IsDir() {
|
|
||||||
t.Fatalf("%s: %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(stderr.String(), "debug=") {
|
|
||||||
t.Fatalf("stderr does not include bundle path: %q", stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunWithoutDebugDoesNotAllocateDebugRoot(t *testing.T) {
|
|
||||||
root := filepath.Join(t.TempDir(), "not-created")
|
|
||||||
configPath := writeV3Config(t, "")
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
lookup := func(name string) (string, bool) {
|
|
||||||
if name == "NOTARIUS_DEBUG_DIR" {
|
|
||||||
return root, true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", "source.json", "--chunk_cache", "bypass"}, &stdout, &stderr, Options{LookupEnv: lookup, RunIDGenerator: stateSurfaceRunIDGenerator})
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(root); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("debug root exists or unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigValidateUsesVersion3AndRemovedFieldsFail(t *testing.T) {
|
|
||||||
configPath := writeV3Config(t, "")
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
if code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{LookupEnv: emptyLookup}); code != 0 {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
legacy := filepath.Join(t.TempDir(), "legacy.yml")
|
|
||||||
if err := os.WriteFile(legacy, []byte("version: 3\nworkspace:\n directory: /tmp/old\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
stdout.Reset()
|
|
||||||
stderr.Reset()
|
|
||||||
if code := RunWithOptions([]string{"config", "validate", "--config", legacy}, &stdout, &stderr, Options{LookupEnv: emptyLookup}); code != 1 || !strings.Contains(stderr.String(), "field workspace not found") {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeV3Config(t *testing.T, extra string) string {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
|
||||||
data := "version: 3\noutput:\n directory: ./out\ncache:\n chunk_plans:\n mode: bypass\n checkpoints: {}\ndebug:\n directory: ./debug\n" + extra + "pipelines: {}\n"
|
|
||||||
if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
func emptyLookup(string) (string, bool) { return "", false }
|
|
||||||
@@ -71,6 +71,11 @@ func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
|
|||||||
yaml: "version: 3\nllm_profiles: {}\n",
|
yaml: "version: 3\nllm_profiles: {}\n",
|
||||||
want: "field llm_profiles not found",
|
want: "field llm_profiles not found",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "version 2 migration",
|
||||||
|
yaml: "version: 2\nworkspace:\n directory: /tmp/old\n",
|
||||||
|
want: "version 2-to-3 migration",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "pipeline field",
|
name: "pipeline field",
|
||||||
yaml: "version: 3\npipelines:\n main:\n unknown: true\n",
|
yaml: "version: 3\npipelines:\n main:\n unknown: true\n",
|
||||||
|
|||||||
@@ -1,124 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestVersion3DefaultsAndValidation(t *testing.T) {
|
|
||||||
cfg := Default()
|
|
||||||
if cfg.Output.Directory != "./notarius-output" || cfg.Debug.Directory != "./notarius-debug" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto {
|
|
||||||
t.Fatalf("unexpected defaults: %#v", cfg)
|
|
||||||
}
|
|
||||||
if err := cfg.Validate(); err != nil {
|
|
||||||
t.Fatalf("Validate() error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVersion3FileSchemaIsStrictAndRejectsVersion2BeforeDecode(t *testing.T) {
|
|
||||||
_, err := ParseFileConfigYAML([]byte("version: 2\nworkspace:\n directory: /tmp/old\n"))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "version 2-to-3 migration") {
|
|
||||||
t.Fatalf("version 2 error = %v", err)
|
|
||||||
}
|
|
||||||
_, err = ParseFileConfigYAML([]byte("version: 3\nworkspace:\n directory: /tmp/old\n"))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "field workspace not found") {
|
|
||||||
t.Fatalf("unknown field error = %v", err)
|
|
||||||
}
|
|
||||||
_, err = ParseFileConfigYAML([]byte("version: 4\n"))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "unsupported config version 4") {
|
|
||||||
t.Fatalf("version 4 error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatePrecedenceAndInvalidSources(t *testing.T) {
|
|
||||||
file, err := ParseFileConfigYAML([]byte(`version: 3
|
|
||||||
output:
|
|
||||||
directory: ./file-output
|
|
||||||
cache:
|
|
||||||
chunk_plans:
|
|
||||||
directory: ./plans
|
|
||||||
mode: refresh
|
|
||||||
checkpoints:
|
|
||||||
directory: ./checkpoints
|
|
||||||
debug:
|
|
||||||
directory: ./debug
|
|
||||||
`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg := Default()
|
|
||||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
lookup := func(name string) (string, bool) {
|
|
||||||
values := map[string]string{
|
|
||||||
"NOTARIUS_OUTPUT_DIR": "/env/output", "NOTARIUS_CACHE_CHUNK_PLANS_MODE": "auto",
|
|
||||||
"NOTARIUS_CACHE_CHUNK_PLANS_DIR": "/env/plans", "NOTARIUS_CACHE_CHECKPOINTS_DIR": "/env/checkpoints", "NOTARIUS_DEBUG_DIR": "/env/debug",
|
|
||||||
}
|
|
||||||
v, ok := values[name]
|
|
||||||
return v, ok
|
|
||||||
}
|
|
||||||
if err := cfg.ApplyEnvOverridesWithLookup(lookup); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if cfg.Output.Directory != "/env/output" || cfg.Cache.ChunkPlans.Directory != "/env/plans" || cfg.Cache.Checkpoints.Directory != "/env/checkpoints" || cfg.Debug.Directory != "/env/debug" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto {
|
|
||||||
t.Fatalf("unexpected environment precedence: %#v", cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
bad := Default()
|
|
||||||
err = bad.ApplyEnvOverridesWithLookup(func(name string) (string, bool) {
|
|
||||||
if name == "NOTARIUS_DEBUG_DIR" {
|
|
||||||
return " ", true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "NOTARIUS_DEBUG_DIR") {
|
|
||||||
t.Fatalf("empty debug environment error = %v", err)
|
|
||||||
}
|
|
||||||
invalidFile, err := ParseFileConfigYAML([]byte("version: 3\noutput:\n directory: ' '\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
bad = Default()
|
|
||||||
if err := bad.ApplyFileConfig(invalidFile); err == nil || !strings.Contains(err.Error(), "output.directory") {
|
|
||||||
t.Fatalf("invalid file error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedactedSummaryContainsOnlyVersion3StateFields(t *testing.T) {
|
|
||||||
cfg := Default()
|
|
||||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{Input: pipeline.ModuleBinding{Module: "input", Options: map[string]any{"api_key": "secret-value", "safe": "value"}}}
|
|
||||||
payload, err := json.Marshal(cfg.RedactedSummaryPayload())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
text := string(payload)
|
|
||||||
for _, forbidden := range []string{"workspace", "diagnostics"} {
|
|
||||||
if strings.Contains(text, forbidden) {
|
|
||||||
t.Fatalf("payload contains %q: %s", forbidden, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(text, "secret-value") || !strings.Contains(text, "[REDACTED]") {
|
|
||||||
t.Fatalf("payload did not redact sensitive option: %s", text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCacheFamilyDefaultsAreIndependent(t *testing.T) {
|
|
||||||
base := filepath.Join(t.TempDir(), "cache")
|
|
||||||
resolver := func() (string, error) { return base, nil }
|
|
||||||
plans, err := DefaultChunkPlanRoot(resolver)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
checkpoints, err := DefaultCheckpointRoot(resolver)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if plans == checkpoints || plans != filepath.Join(base, "notarius", "chunk-plans") || checkpoints != filepath.Join(base, "notarius", "checkpoints") {
|
|
||||||
t.Fatalf("roots = %q, %q", plans, checkpoints)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -17,44 +17,39 @@ func TestValidateDocumentValid(t *testing.T) {
|
|||||||
func TestValidateDocumentNil(t *testing.T) {
|
func TestValidateDocumentNil(t *testing.T) {
|
||||||
err := ValidateDocument(nil)
|
err := ValidateDocument(nil)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, "source document", "nil")
|
||||||
t.Fatal("ValidateDocument() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != "source document must not be nil" {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q", err.Error())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDocumentMissingFields(t *testing.T) {
|
func TestValidateDocumentMissingFields(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*SourceDocument)
|
mutate func(*SourceDocument)
|
||||||
wantErr string
|
fragments []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "id",
|
name: "id",
|
||||||
mutate: func(doc *SourceDocument) { doc.ID = " \t" },
|
mutate: func(doc *SourceDocument) { doc.ID = " \t" },
|
||||||
wantErr: "source document id must not be empty",
|
fragments: []string{"source document id", "must not be empty"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "id surrounding whitespace",
|
name: "id surrounding whitespace",
|
||||||
mutate: func(doc *SourceDocument) { doc.ID = " source-1 " },
|
mutate: func(doc *SourceDocument) { doc.ID = " source-1 " },
|
||||||
wantErr: "source document id \" source-1 \" must not contain leading or trailing whitespace",
|
fragments: []string{"source document id", "leading or trailing whitespace"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "kind",
|
name: "kind",
|
||||||
mutate: func(doc *SourceDocument) { doc.Kind = "" },
|
mutate: func(doc *SourceDocument) { doc.Kind = "" },
|
||||||
wantErr: "source document kind must not be empty",
|
fragments: []string{"source document kind", "must not be empty"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "format",
|
name: "format",
|
||||||
mutate: func(doc *SourceDocument) { doc.Format = "\n" },
|
mutate: func(doc *SourceDocument) { doc.Format = "\n" },
|
||||||
wantErr: "source document format must not be empty",
|
fragments: []string{"source document format", "must not be empty"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "digest",
|
name: "digest",
|
||||||
mutate: func(doc *SourceDocument) { doc.Digest = "" },
|
mutate: func(doc *SourceDocument) { doc.Digest = "" },
|
||||||
wantErr: "source document digest must not be empty",
|
fragments: []string{"source document digest", "must not be empty"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,12 +60,7 @@ func TestValidateDocumentMissingFields(t *testing.T) {
|
|||||||
|
|
||||||
err := ValidateDocument(doc)
|
err := ValidateDocument(doc)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, tt.fragments...)
|
||||||
t.Fatal("ValidateDocument() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != tt.wantErr {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -81,34 +71,29 @@ func TestValidateDocumentEmptyUnits(t *testing.T) {
|
|||||||
|
|
||||||
err := ValidateDocument(doc)
|
err := ValidateDocument(doc)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, "source document units", "must not be empty")
|
||||||
t.Fatal("ValidateDocument() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != "source document units must not be empty" {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q", err.Error())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDocumentMissingUnitFields(t *testing.T) {
|
func TestValidateDocumentMissingUnitFields(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*SourceDocument)
|
mutate func(*SourceDocument)
|
||||||
wantErr string
|
fragments []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "id",
|
name: "id",
|
||||||
mutate: func(doc *SourceDocument) { doc.Units[1].ID = 0 },
|
mutate: func(doc *SourceDocument) { doc.Units[1].ID = 0 },
|
||||||
wantErr: "source unit[1].id must be positive",
|
fragments: []string{"source unit[1].id", "must be positive"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "kind",
|
name: "kind",
|
||||||
mutate: func(doc *SourceDocument) { doc.Units[1].Kind = " " },
|
mutate: func(doc *SourceDocument) { doc.Units[1].Kind = " " },
|
||||||
wantErr: "source unit[1].kind must not be empty",
|
fragments: []string{"source unit[1].kind", "must not be empty"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "text",
|
name: "text",
|
||||||
mutate: func(doc *SourceDocument) { doc.Units[1].Text = "\n\t" },
|
mutate: func(doc *SourceDocument) { doc.Units[1].Text = "\n\t" },
|
||||||
wantErr: "source unit[1].text must not be empty",
|
fragments: []string{"source unit[1].text", "must not be empty"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,12 +104,7 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
|
|||||||
|
|
||||||
err := ValidateDocument(doc)
|
err := ValidateDocument(doc)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, tt.fragments...)
|
||||||
t.Fatal("ValidateDocument() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != tt.wantErr {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -135,45 +115,27 @@ func TestValidateDocumentDuplicateUnitIDs(t *testing.T) {
|
|||||||
|
|
||||||
err := ValidateDocument(doc)
|
err := ValidateDocument(doc)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, "source unit id 1", "duplicated")
|
||||||
t.Fatal("ValidateDocument() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != "source unit id 1 is duplicated" {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q", err.Error())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDocumentUnitReferences(t *testing.T) {
|
func TestValidateDocumentUnitReferences(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*SourceDocument)
|
mutate func(*SourceDocument)
|
||||||
wantErr string
|
fragments []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "missing",
|
name: "nested reference context",
|
||||||
mutate: func(doc *SourceDocument) { doc.Units[0].Ref = SourceRef{} },
|
|
||||||
wantErr: "source unit[0].ref: source ref source_id must not be empty",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "foreign source",
|
|
||||||
mutate: func(doc *SourceDocument) { doc.Units[0].Ref.SourceID = "source-2" },
|
mutate: func(doc *SourceDocument) { doc.Units[0].Ref.SourceID = "source-2" },
|
||||||
wantErr: "source unit[0].ref: source ref source_id \"source-2\" does not match document id \"source-1\"",
|
fragments: []string{"source unit[0].ref", "source_id", "does not match"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "non-self range",
|
name: "document unit self-reference",
|
||||||
mutate: func(doc *SourceDocument) {
|
mutate: func(doc *SourceDocument) {
|
||||||
doc.Units[0].Ref.StartUnitID = 2
|
doc.Units[0].Ref.StartUnitID = 2
|
||||||
doc.Units[0].Ref.EndUnitID = 2
|
doc.Units[0].Ref.EndUnitID = 2
|
||||||
},
|
},
|
||||||
wantErr: "source unit[0].ref must identify source unit id 1",
|
fragments: []string{"source unit[0].ref", "must identify source unit id 1"},
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "reversed range",
|
|
||||||
mutate: func(doc *SourceDocument) {
|
|
||||||
doc.Units[0].Ref.StartUnitID = 2
|
|
||||||
doc.Units[0].Ref.EndUnitID = 1
|
|
||||||
},
|
|
||||||
wantErr: "source unit[0].ref: source ref start_unit_id 2 appears after end_unit_id 1",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,12 +145,7 @@ func TestValidateDocumentUnitReferences(t *testing.T) {
|
|||||||
tt.mutate(doc)
|
tt.mutate(doc)
|
||||||
|
|
||||||
err := ValidateDocument(doc)
|
err := ValidateDocument(doc)
|
||||||
if err == nil {
|
requireErrorFragments(t, err, tt.fragments...)
|
||||||
t.Fatal("ValidateDocument() error = nil, want unit reference error")
|
|
||||||
}
|
|
||||||
if err.Error() != tt.wantErr {
|
|
||||||
t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -294,59 +251,51 @@ func TestValidateRefValid(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateRefSourceIDMismatch(t *testing.T) {
|
func TestValidateRefRejectsMalformedReferences(t *testing.T) {
|
||||||
doc := validDocument()
|
|
||||||
ref := SourceRef{
|
|
||||||
SourceID: "source-2",
|
|
||||||
StartUnitID: 1,
|
|
||||||
EndUnitID: 2,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := ValidateRef(doc, ref)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("ValidateRef() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != "source ref source_id \"source-2\" does not match document id \"source-1\"" {
|
|
||||||
t.Fatalf("ValidateRef() error = %q", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateRefMissingUnitIDs(t *testing.T) {
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
ref SourceRef
|
ref SourceRef
|
||||||
wantErr string
|
fragments []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "missing source id",
|
name: "missing source id",
|
||||||
ref: SourceRef{StartUnitID: 1, EndUnitID: 2},
|
ref: SourceRef{StartUnitID: 1, EndUnitID: 2},
|
||||||
wantErr: "source ref source_id must not be empty",
|
fragments: []string{"source_id", "must not be empty"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "source id surrounding whitespace",
|
name: "source id surrounding whitespace",
|
||||||
ref: SourceRef{SourceID: " source-1 ", StartUnitID: 1, EndUnitID: 2},
|
ref: SourceRef{SourceID: " source-1 ", StartUnitID: 1, EndUnitID: 2},
|
||||||
wantErr: "source ref source_id \" source-1 \" must not contain leading or trailing whitespace",
|
fragments: []string{"source_id", "leading or trailing whitespace"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing start id",
|
name: "missing start id",
|
||||||
ref: SourceRef{SourceID: "source-1", EndUnitID: 2},
|
ref: SourceRef{SourceID: "source-1", EndUnitID: 2},
|
||||||
wantErr: "source ref start_unit_id must be positive",
|
fragments: []string{"start_unit_id", "must be positive"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing end id",
|
name: "missing end id",
|
||||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 1},
|
ref: SourceRef{SourceID: "source-1", StartUnitID: 1},
|
||||||
wantErr: "source ref end_unit_id must be positive",
|
fragments: []string{"end_unit_id", "must be positive"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "unknown start id",
|
name: "unknown start id",
|
||||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 2},
|
ref: SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 2},
|
||||||
wantErr: "source ref start_unit_id 9 was not found",
|
fragments: []string{"start_unit_id", "was not found"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "unknown end id",
|
name: "unknown end id",
|
||||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 9},
|
ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 9},
|
||||||
wantErr: "source ref end_unit_id 9 was not found",
|
fragments: []string{"end_unit_id", "was not found"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source id mismatch",
|
||||||
|
ref: SourceRef{SourceID: "source-2", StartUnitID: 1, EndUnitID: 2},
|
||||||
|
fragments: []string{"source_id", "does not match"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "reversed unit order",
|
||||||
|
ref: SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 1},
|
||||||
|
fragments: []string{"start_unit_id", "appears after"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -354,34 +303,11 @@ func TestValidateRefMissingUnitIDs(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
err := ValidateRef(validDocument(), tt.ref)
|
err := ValidateRef(validDocument(), tt.ref)
|
||||||
|
|
||||||
if err == nil {
|
requireErrorFragments(t, err, tt.fragments...)
|
||||||
t.Fatal("ValidateRef() error = nil, want error")
|
|
||||||
}
|
|
||||||
if err.Error() != tt.wantErr {
|
|
||||||
t.Fatalf("ValidateRef() error = %q, want %q", err.Error(), tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateRefReversedUnitOrder(t *testing.T) {
|
|
||||||
doc := validDocument()
|
|
||||||
ref := SourceRef{
|
|
||||||
SourceID: "source-1",
|
|
||||||
StartUnitID: 2,
|
|
||||||
EndUnitID: 1,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := ValidateRef(doc, ref)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("ValidateRef() error = nil, want error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "appears after") {
|
|
||||||
t.Fatalf("ValidateRef() error = %q, want reversed order error", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnitIndex(t *testing.T) {
|
func TestUnitIndex(t *testing.T) {
|
||||||
doc := validDocument()
|
doc := validDocument()
|
||||||
|
|
||||||
@@ -424,3 +350,15 @@ func validDocument() *SourceDocument {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requireErrorFragments(t *testing.T, err error, fragments ...string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want fragments %q", fragments)
|
||||||
|
}
|
||||||
|
for _, fragment := range fragments {
|
||||||
|
if !strings.Contains(err.Error(), fragment) {
|
||||||
|
t.Fatalf("error = %q, want fragment %q", err.Error(), fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
415
internal/framework/checkpoint/filesystem_test.go
Normal file
415
internal/framework/checkpoint/filesystem_test.go
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
package checkpoint
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
type filesystemCheckpointFixture struct {
|
||||||
|
root string
|
||||||
|
identity Identity
|
||||||
|
loader pipeline.CheckpointLoader
|
||||||
|
doc source.SourceDocument
|
||||||
|
extract pipeline.CheckpointArtifact
|
||||||
|
merge pipeline.CheckpointArtifact
|
||||||
|
normalize pipeline.CheckpointArtifact
|
||||||
|
dependencies []pipeline.CheckpointFingerprint
|
||||||
|
warnings []contracts.Warning
|
||||||
|
rejected []contracts.RejectedOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
|
||||||
|
// Recording owns its inputs. These mutations must not change the durable values.
|
||||||
|
fixture.doc.Units[0].Text = "caller mutation"
|
||||||
|
fixture.doc.Metadata["owner"] = "caller mutation"
|
||||||
|
fixture.extract.Artifact.Content[0] = 'x'
|
||||||
|
fixture.extract.Artifact.Metadata["content"] = "caller mutation"
|
||||||
|
fixture.merge.Artifact.Content[0] = 'x'
|
||||||
|
fixture.normalize.Artifact.Content[0] = 'x'
|
||||||
|
fixture.warnings[0].Message = "caller mutation"
|
||||||
|
fixture.rejected[0].Message = "caller mutation"
|
||||||
|
|
||||||
|
t.Run("source", func(t *testing.T) {
|
||||||
|
got, decision := fixture.loader.Source("source-module")
|
||||||
|
if !decision.Reused {
|
||||||
|
t.Fatalf("source decision = %#v", decision)
|
||||||
|
}
|
||||||
|
if got.Document == nil || got.Document.ID != "document-1" || got.Document.Units[0].Text != "original source" || got.Document.Metadata["owner"] != "fixture" {
|
||||||
|
t.Fatalf("source was not restored: %#v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
got.Document.Units[0].Text = "loaded mutation"
|
||||||
|
got.Document.Metadata["owner"] = "loaded mutation"
|
||||||
|
reloaded, decision := fixture.loader.Source("source-module")
|
||||||
|
if !decision.Reused || reloaded.Document.Units[0].Text != "original source" || reloaded.Document.Metadata["owner"] != "fixture" {
|
||||||
|
t.Fatalf("source reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("extract", func(t *testing.T) {
|
||||||
|
got, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||||
|
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Warnings) != 1 {
|
||||||
|
t.Fatalf("extract result=%#v decision=%#v", got, decision)
|
||||||
|
}
|
||||||
|
output := got.Outputs[0]
|
||||||
|
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || got.Warnings[0].ReasonCode != "partial" || got.Rejected[0].ReasonCode != "invalid_source" {
|
||||||
|
t.Fatalf("extract values were not restored: %#v", got)
|
||||||
|
}
|
||||||
|
manifest := readManifest[ExtractLaneManifest](t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "manifest.json"))
|
||||||
|
if manifest.Status != StatusSucceededWithRejections {
|
||||||
|
t.Fatalf("extract status = %q, want succeeded with rejections", manifest.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
got.Outputs[0].Artifact.Content[0] = 'y'
|
||||||
|
got.Warnings[0].Message = "loaded mutation"
|
||||||
|
reloaded, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||||
|
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) || reloaded.Warnings[0].Message != "partial output" {
|
||||||
|
t.Fatalf("extract reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
load func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision)
|
||||||
|
want []byte
|
||||||
|
}{
|
||||||
|
{name: "merge", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||||
|
got, decision := fixture.loader.Merge("lane-a", "merge-module", fixture.dependencies)
|
||||||
|
return got.Output, got.Warnings, decision
|
||||||
|
}, want: []byte(`{"spells":["fire"]}`)},
|
||||||
|
{name: "normalize", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||||
|
got, decision := fixture.loader.Normalize("lane-a", "normalize-module", fixture.dependencies)
|
||||||
|
return got.Output, got.Warnings, decision
|
||||||
|
}, want: []byte(`{"spells":["fire"],"normalized":true}`)},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, warnings, decision := tt.load()
|
||||||
|
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(warnings) != 1 || warnings[0].ReasonCode != "review" {
|
||||||
|
t.Fatalf("%s result=%#v warnings=%#v decision=%#v", tt.name, got, warnings, decision)
|
||||||
|
}
|
||||||
|
|
||||||
|
got.Artifact.Content[0] = 'z'
|
||||||
|
reloaded, warnings, decision := tt.load()
|
||||||
|
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) || warnings[0].Message != "review manually" {
|
||||||
|
t.Fatalf("%s reload changed after loaded mutation: %#v warnings=%#v decision=%#v", tt.name, reloaded, warnings, decision)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
t.Run("restrictive permissions", func(t *testing.T) {
|
||||||
|
root := filepath.Join(fixture.root, mustRelativePath(t, fixture.identity))
|
||||||
|
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
want := os.FileMode(0o600)
|
||||||
|
if info.IsDir() {
|
||||||
|
want = 0o700
|
||||||
|
}
|
||||||
|
if got := info.Mode().Perm(); got != want {
|
||||||
|
t.Errorf("%s permissions = %o, want %o", path, got, want)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRejectsMissingAndCorruptState(t *testing.T) {
|
||||||
|
for _, stage := range checkpointStages() {
|
||||||
|
t.Run(stage.name+" missing manifest", func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
if err := os.Remove(stage.manifest(fixture)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertStageNotReused(t, stage, fixture, "missing")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run(stage.name+" malformed manifest", func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
if err := os.WriteFile(stage.manifest(fixture), []byte("{"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertStageNotReused(t, stage, fixture, "decode")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
edit func(map[string]any)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, "workspace schema"},
|
||||||
|
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, "workspace schema"},
|
||||||
|
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, "identity"},
|
||||||
|
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, "stage"},
|
||||||
|
{"lane", func(m map[string]any) { m["lane_id"] = "lane-other" }, "lane"},
|
||||||
|
{"module", func(m map[string]any) { m["module_key"] = "module-other" }, "module"},
|
||||||
|
{"dependency", func(m map[string]any) {
|
||||||
|
m["dependency_fingerprints"] = []map[string]string{{"name": "input", "value": "other"}}
|
||||||
|
}, "dependency"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editManifest(t, checkpointStages()[1].manifest(fixture), tt.edit)
|
||||||
|
assertStageNotReused(t, checkpointStages()[1], fixture, tt.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRejectsNonTerminalStatuses(t *testing.T) {
|
||||||
|
for _, status := range []StageStatus{StatusRunning, StatusFailed, StatusPending, StatusInvalidated} {
|
||||||
|
t.Run(string(status), func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) { m["status"] = string(status) })
|
||||||
|
assertStageNotReused(t, checkpointStages()[1], fixture, "status")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
edit func(map[string]any)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"artifact kind", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = "" }, "artifact codec identity"},
|
||||||
|
{"schema id", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["id"] = "" }, "artifact codec identity"},
|
||||||
|
{"schema version", func(m map[string]any) {
|
||||||
|
m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["version"] = ""
|
||||||
|
}, "artifact codec identity"},
|
||||||
|
{"schema digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema_digest"] = "" }, "artifact codec identity"},
|
||||||
|
{"base64", func(m map[string]any) {
|
||||||
|
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_base64"] = "%"
|
||||||
|
}, "base64"},
|
||||||
|
{"content digest", func(m map[string]any) {
|
||||||
|
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other"
|
||||||
|
}, "content digest"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "outputs.json"), tt.edit)
|
||||||
|
assertStageNotReused(t, checkpointStages()[1], fixture, tt.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T) {
|
||||||
|
t.Run("invalid source document", func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
|
||||||
|
m["document"].(map[string]any)["units"].([]any)[0].(map[string]any)["text"] = ""
|
||||||
|
})
|
||||||
|
assertStageNotReused(t, checkpointStages()[0], fixture, "source checkpoint document")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("source output digest", func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
|
||||||
|
m["document"].(map[string]any)["digest"] = "sha256:other"
|
||||||
|
})
|
||||||
|
assertStageNotReused(t, checkpointStages()[0], fixture, "output digest")
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, stage := range checkpointStages()[1:] {
|
||||||
|
t.Run(stage.name+" output digest", func(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
editManifest(t, stage.manifest(fixture), func(m map[string]any) {
|
||||||
|
m["output_digests"].([]any)[0].(map[string]any)["value"] = "sha256:other"
|
||||||
|
})
|
||||||
|
assertStageNotReused(t, stage, fixture, "output digest")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemCheckpointReusesExtractWithRejections(t *testing.T) {
|
||||||
|
fixture := seedFilesystemCheckpoints(t)
|
||||||
|
result, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||||
|
if !decision.Reused || len(result.Rejected) != 1 || result.Rejected[0].Message != "source reference is invalid" {
|
||||||
|
t.Fatalf("result=%#v decision=%#v", result, decision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type checkpointStage struct {
|
||||||
|
name string
|
||||||
|
manifest func(filesystemCheckpointFixture) string
|
||||||
|
load func(filesystemCheckpointFixture) pipeline.CheckpointDecision
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkpointStages() []checkpointStage {
|
||||||
|
return []checkpointStage{
|
||||||
|
{
|
||||||
|
name: "source",
|
||||||
|
manifest: func(f filesystemCheckpointFixture) string {
|
||||||
|
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "source", "manifest.json")
|
||||||
|
},
|
||||||
|
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||||
|
_, d := f.loader.Source("source-module")
|
||||||
|
return d
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract",
|
||||||
|
manifest: func(f filesystemCheckpointFixture) string {
|
||||||
|
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "extract", "lane-a", "manifest.json")
|
||||||
|
},
|
||||||
|
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||||
|
_, d := f.loader.Extract("lane-a", "extract-module", f.dependencies)
|
||||||
|
return d
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "merge",
|
||||||
|
manifest: func(f filesystemCheckpointFixture) string {
|
||||||
|
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "merge", "lane-a", "manifest.json")
|
||||||
|
},
|
||||||
|
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||||
|
_, d := f.loader.Merge("lane-a", "merge-module", f.dependencies)
|
||||||
|
return d
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize",
|
||||||
|
manifest: func(f filesystemCheckpointFixture) string {
|
||||||
|
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "normalize", "lane-a", "manifest.json")
|
||||||
|
},
|
||||||
|
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||||
|
_, d := f.loader.Normalize("lane-a", "normalize-module", f.dependencies)
|
||||||
|
return d
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedFilesystemCheckpoints(t *testing.T) filesystemCheckpointFixture {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
identity, err := NewIdentity(representativeIdentityInput())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
recorder, err := NewFilesystemRecorder(root, identity)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fixture := filesystemCheckpointFixture{
|
||||||
|
root: root,
|
||||||
|
identity: identity,
|
||||||
|
doc: checkpointDocument(),
|
||||||
|
extract: checkpointArtifact("extract", `{"spell":"fire"}`),
|
||||||
|
merge: checkpointArtifact("merge", `{"spells":["fire"]}`),
|
||||||
|
normalize: checkpointArtifact("normalize", `{"spells":["fire"],"normalized":true}`),
|
||||||
|
dependencies: []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:source"}, {Name: "chunk-plan", Value: "sha256:plan"}},
|
||||||
|
warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "partial", Message: "partial output"}},
|
||||||
|
rejected: []contracts.RejectedOutput{{Stage: "extract", LaneID: "lane-a", ModuleKey: "extract-module", ChunkID: "chunk-a", ValidatorName: "source_refs", ReasonCode: "invalid_source", Message: "source reference is invalid", AttemptCount: 1}},
|
||||||
|
}
|
||||||
|
if err := recorder.SourceSucceeded("source-module", &fixture.doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected, fixture.warnings); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mergeWarnings := []contracts.Warning{{Scope: "merge", ReasonCode: "review", Message: "review manually"}}
|
||||||
|
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge, mergeWarnings); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize, mergeWarnings); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fixture.loader, err = NewFilesystemLoader(root, identity)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return fixture
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkpointDocument() source.SourceDocument {
|
||||||
|
return source.SourceDocument{
|
||||||
|
ID: "document-1", Kind: "transcript", Format: "text", Digest: "sha256:document", Metadata: map[string]any{"owner": "fixture", "number": float64(1)},
|
||||||
|
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "original source", Ref: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "narrator"}}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkpointArtifact(module, content string) pipeline.CheckpointArtifact {
|
||||||
|
return pipeline.CheckpointArtifact{
|
||||||
|
LaneID: "lane-a", ModuleKey: module, SourceID: "document-1", ChunkID: "chunk-a", ChunkIndex: 0,
|
||||||
|
ChunkRef: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
|
||||||
|
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStageNotReused(t *testing.T, stage checkpointStage, fixture filesystemCheckpointFixture, want string) {
|
||||||
|
t.Helper()
|
||||||
|
decision := stage.load(fixture)
|
||||||
|
if decision.Reused || !strings.Contains(strings.ToLower(decision.Reason), strings.ToLower(want)) {
|
||||||
|
t.Fatalf("%s decision=%#v, want non-reused reason containing %q", stage.name, decision, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func editManifest(t *testing.T, path string, edit func(map[string]any)) {
|
||||||
|
t.Helper()
|
||||||
|
editJSON(t, path, edit)
|
||||||
|
}
|
||||||
|
|
||||||
|
func editJSON(t *testing.T, path string, edit func(map[string]any)) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var value map[string]any
|
||||||
|
if err := json.Unmarshal(data, &value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
edit(value)
|
||||||
|
data, err = json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readManifest[T any](t *testing.T, path string) T {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var value T
|
||||||
|
if err := json.Unmarshal(data, &value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRelativePath(t *testing.T, identity Identity) string {
|
||||||
|
t.Helper()
|
||||||
|
return mustRelativePathForTest(identity)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRelativePathForTest(identity Identity) string {
|
||||||
|
path, err := identity.RelativePath()
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
202
internal/framework/checkpoint/identity_test.go
Normal file
202
internal/framework/checkpoint/identity_test.go
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
package checkpoint
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewIdentityNormalizesOrderAndEmptyValues(t *testing.T) {
|
||||||
|
base := representativeIdentityInput()
|
||||||
|
identity, err := NewIdentity(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*IdentityInput)
|
||||||
|
}{
|
||||||
|
{"selected lanes", func(v *IdentityInput) { v.SelectedLanes = []string{"lane-a", "lane-b"} }},
|
||||||
|
{"runtime fingerprints", func(v *IdentityInput) {
|
||||||
|
v.RuntimeOverrides = []Fingerprint{{Name: "model", Value: "large"}, {Name: "timeout", Value: "30s"}}
|
||||||
|
}},
|
||||||
|
{"references", func(v *IdentityInput) {
|
||||||
|
v.References = []artifacts.ReferenceProvenance{v.References[1], v.References[0]}
|
||||||
|
}},
|
||||||
|
{"provenance fingerprints", func(v *IdentityInput) {
|
||||||
|
v.ProvenanceFingerprints = []Fingerprint{{Name: "source", Value: "v2"}, {Name: "runner", Value: "v1"}}
|
||||||
|
}},
|
||||||
|
{"resolved lanes", func(v *IdentityInput) {
|
||||||
|
v.Pipeline.ArtifactLanes = []pipeline.ResolvedArtifactLane{v.Pipeline.ArtifactLanes[1], v.Pipeline.ArtifactLanes[0]}
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
changed := cloneIdentityInput(base)
|
||||||
|
tt.mutate(&changed)
|
||||||
|
got, err := NewIdentity(changed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(identity, got) {
|
||||||
|
t.Fatalf("reordered identity differs:\nbase=%#v\ngot=%#v", identity, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("duplicates and blanks are ignored", func(t *testing.T) {
|
||||||
|
changed := base
|
||||||
|
changed.SelectedLanes = []string{" ", "lane-b", "lane-a", "lane-a", ""}
|
||||||
|
changed.RuntimeOverrides = append(changed.RuntimeOverrides, Fingerprint{}, Fingerprint{Name: " ", Value: "ignored"}, Fingerprint{Name: "timeout", Value: "30s"})
|
||||||
|
changed.ProvenanceFingerprints = append(changed.ProvenanceFingerprints, Fingerprint{}, Fingerprint{Name: "", Value: "ignored"})
|
||||||
|
|
||||||
|
got, err := NewIdentity(changed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(identity, got) {
|
||||||
|
t.Fatalf("empty or duplicate values changed identity:\nbase=%#v\ngot=%#v", identity, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewIdentityChangesForMeaningfulInputs(t *testing.T) {
|
||||||
|
base := representativeIdentityInput()
|
||||||
|
original, err := NewIdentity(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := map[string]func(*IdentityInput){
|
||||||
|
"pipeline id": func(v *IdentityInput) { v.Pipeline.ID = "another-pipeline" },
|
||||||
|
"pipeline digest": func(v *IdentityInput) { v.Pipeline.Digest = "sha256:pipeline-digest-2" },
|
||||||
|
"input key": func(v *IdentityInput) { v.InputKey = "another-input" },
|
||||||
|
"raw input digest": func(v *IdentityInput) { v.RawInputDigest = "sha256:raw-input-2" },
|
||||||
|
"source digest": func(v *IdentityInput) { v.SourceDigest = "sha256:source-2" },
|
||||||
|
"selected lanes": func(v *IdentityInput) { v.SelectedLanes = []string{"lane-a"} },
|
||||||
|
"runtime override": func(v *IdentityInput) { v.RuntimeOverrides[0].Value = "60s" },
|
||||||
|
"reference digest": func(v *IdentityInput) { v.References[0].Digest = "sha256:reference-2" },
|
||||||
|
"reference identity": func(v *IdentityInput) { v.References[0].OriginURI = "file:///other-reference" },
|
||||||
|
"provenance": func(v *IdentityInput) { v.ProvenanceFingerprints[0].Value = "v3" },
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, mutate := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
changed := cloneIdentityInput(base)
|
||||||
|
mutate(&changed)
|
||||||
|
got, err := NewIdentity(changed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Digest == original.Digest {
|
||||||
|
t.Fatalf("meaningful %s input did not change digest %q", name, got.Digest)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewIdentityUsesResolvedInputWhenKeyIsOmitted(t *testing.T) {
|
||||||
|
input := representativeIdentityInput()
|
||||||
|
input.InputKey = ""
|
||||||
|
|
||||||
|
identity, err := NewIdentity(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if identity.InputKey != input.Pipeline.Input.Module {
|
||||||
|
t.Fatalf("input key = %q, want resolved module %q", identity.InputKey, input.Pipeline.Input.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewIdentityRejectsMissingRequiredInputs(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*IdentityInput)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"pipeline id", func(v *IdentityInput) { v.Pipeline.ID = "" }, "pipeline id"},
|
||||||
|
{"pipeline digest", func(v *IdentityInput) { v.Pipeline.Digest = "" }, "pipeline digest"},
|
||||||
|
{"input key", func(v *IdentityInput) { v.InputKey = ""; v.Pipeline.Input = pipeline.Binding("") }, "input key"},
|
||||||
|
{"input digests", func(v *IdentityInput) { v.RawInputDigest = ""; v.SourceDigest = "" }, "raw input digest or source digest"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range cases {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
input := representativeIdentityInput()
|
||||||
|
tt.mutate(&input)
|
||||||
|
_, err := NewIdentity(input)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("error = %v, want category containing %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentityRelativePathIsDeterministicAndConfined(t *testing.T) {
|
||||||
|
identity, err := NewIdentity(representativeIdentityInput())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := identity.RelativePath()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := identity.RelativePath()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first != second {
|
||||||
|
t.Fatalf("relative path is not deterministic: %q != %q", first, second)
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(first) || filepath.Clean(first) != first || filepath.ToSlash(first) != first {
|
||||||
|
t.Fatalf("path is not a clean relative slash-separated path: %q", first)
|
||||||
|
}
|
||||||
|
if strings.Contains(first, "../") || strings.HasPrefix(first, "../") || strings.Contains(first, `\\`) {
|
||||||
|
t.Fatalf("path escapes its root: %q", first)
|
||||||
|
}
|
||||||
|
parts := strings.Split(first, "/")
|
||||||
|
if len(parts) != 4 || parts[0] != "pipeline" || !strings.HasPrefix(parts[1], "input-") {
|
||||||
|
t.Fatalf("path does not contain the documented identity hierarchy: %q", first)
|
||||||
|
}
|
||||||
|
if !strings.Contains(parts[1], "source-digest") || !strings.Contains(parts[2], "pipeline-digest") || parts[3] == "" {
|
||||||
|
t.Fatalf("path omits digest-derived hierarchy: %q", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func representativeIdentityInput() IdentityInput {
|
||||||
|
return IdentityInput{
|
||||||
|
Pipeline: pipeline.ResolvedPipeline{
|
||||||
|
ID: "pipeline",
|
||||||
|
Digest: "sha256:pipeline-digest-000000000000",
|
||||||
|
Input: pipeline.Binding("input"),
|
||||||
|
ArtifactLanes: []pipeline.ResolvedArtifactLane{
|
||||||
|
{ID: "lane-b"},
|
||||||
|
{ID: "lane-a"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
InputKey: "input",
|
||||||
|
RawInputDigest: "sha256:raw-input-000000000000",
|
||||||
|
SourceDigest: "sha256:source-digest-000000000000",
|
||||||
|
SelectedLanes: []string{"lane-b", "lane-a"},
|
||||||
|
RuntimeOverrides: []Fingerprint{{Name: "timeout", Value: "30s"}, {Name: "model", Value: "large"}},
|
||||||
|
References: []artifacts.ReferenceProvenance{
|
||||||
|
{Stage: "chunk", SlotName: "glossary", OriginURI: "file:///glossary", Digest: "sha256:reference-1"},
|
||||||
|
{Stage: "extract", LaneID: "lane-a", SlotName: "party", OriginURI: "file:///party", Digest: "sha256:reference-2"},
|
||||||
|
},
|
||||||
|
ProvenanceFingerprints: []Fingerprint{{Name: "runner", Value: "v1"}, {Name: "source", Value: "v2"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneIdentityInput(input IdentityInput) IdentityInput {
|
||||||
|
input.SelectedLanes = append([]string(nil), input.SelectedLanes...)
|
||||||
|
input.RuntimeOverrides = append([]Fingerprint(nil), input.RuntimeOverrides...)
|
||||||
|
input.References = append([]artifacts.ReferenceProvenance(nil), input.References...)
|
||||||
|
input.ProvenanceFingerprints = append([]Fingerprint(nil), input.ProvenanceFingerprints...)
|
||||||
|
input.Pipeline.ArtifactLanes = append([]pipeline.ResolvedArtifactLane(nil), input.Pipeline.ArtifactLanes...)
|
||||||
|
return input
|
||||||
|
}
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
package chunkplan
|
|
||||||
|
|
||||||
import (
|
|
||||||
"go/parser"
|
|
||||||
"go/token"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
const repositoryImportPrefix = "gitea.maximumdirect.net/eric/notarius/internal/"
|
|
||||||
|
|
||||||
func TestPlanStoreAndSourceImportBoundaries(t *testing.T) {
|
|
||||||
repositoryRoot := repositoryRoot(t)
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
directory string
|
|
||||||
forbidden []string
|
|
||||||
}{
|
|
||||||
{name: "source is framework and module independent", directory: "internal/core/source", forbidden: []string{"framework/", "modules/"}},
|
|
||||||
{name: "plan store is module independent", directory: "internal/framework/chunkplan", forbidden: []string{"modules/"}},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
files, err := filepath.Glob(filepath.Join(repositoryRoot, tc.directory, "*.go"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, filename := range files {
|
|
||||||
if strings.HasSuffix(filename, "_test.go") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
parsed, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.ImportsOnly)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, item := range parsed.Imports {
|
|
||||||
path, err := strconv.Unquote(item.Path.Value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path = strings.TrimPrefix(path, repositoryImportPrefix)
|
|
||||||
for _, prefix := range tc.forbidden {
|
|
||||||
if strings.HasPrefix(path, prefix) {
|
|
||||||
t.Fatalf("%s imports %q, forbidden by %s boundary", filepath.Base(filename), path, tc.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func repositoryRoot(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
_, filename, _, ok := runtime.Caller(0)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("resolve test location")
|
|
||||||
}
|
|
||||||
return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..", ".."))
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,155 +17,6 @@ var _ Normalizer[fakeArtifact] = fakeNormalizer{}
|
|||||||
var _ StructuredLLMClient = fakeLLMClient{}
|
var _ StructuredLLMClient = fakeLLMClient{}
|
||||||
var _ OutputEncoder = fakeOutputEncoder{}
|
var _ OutputEncoder = fakeOutputEncoder{}
|
||||||
|
|
||||||
func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
|
|
||||||
extractor := fakeExtractor{
|
|
||||||
key: "generic-extractor",
|
|
||||||
}
|
|
||||||
doc := &source.SourceDocument{
|
|
||||||
ID: "source-1",
|
|
||||||
Kind: "document",
|
|
||||||
Format: "text/plain",
|
|
||||||
Digest: "sha256:abc123",
|
|
||||||
Units: []source.SourceUnit{
|
|
||||||
{ID: 1, Kind: "section", Text: "Source text."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Extract() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if extractor.Key() != "generic-extractor" {
|
|
||||||
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
|
|
||||||
}
|
|
||||||
if result.Value.Value != "example" {
|
|
||||||
t.Fatalf("Value = %q, want example", result.Value.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeChunkerReturnsSourcePlan(t *testing.T) {
|
|
||||||
doc := &source.SourceDocument{
|
|
||||||
ID: "source-1",
|
|
||||||
Kind: "document",
|
|
||||||
Format: "text/plain",
|
|
||||||
Digest: "sha256:abc123",
|
|
||||||
Units: []source.SourceUnit{
|
|
||||||
{ID: 1, Kind: "section", Text: "Source text."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
chunker := fakeChunker{key: "generic-chunker"}
|
|
||||||
|
|
||||||
result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Plan() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if chunker.Key() != "generic-chunker" {
|
|
||||||
t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
|
|
||||||
}
|
|
||||||
if len(result.Plan.Ranges) != 1 {
|
|
||||||
t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges))
|
|
||||||
}
|
|
||||||
if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
|
|
||||||
t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
|
|
||||||
doc := &source.SourceDocument{
|
|
||||||
ID: "source-1",
|
|
||||||
Kind: "document",
|
|
||||||
Format: "text/plain",
|
|
||||||
Digest: "sha256:abc123",
|
|
||||||
Units: []source.SourceUnit{
|
|
||||||
{ID: 1, Kind: "section", Text: "Source text."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
chunker := &recordingChunker{key: "llm-chunker"}
|
|
||||||
|
|
||||||
if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
|
|
||||||
t.Fatalf("Plan() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
|
|
||||||
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
|
||||||
extractor := fakeExtractor{key: "generic-extractor"}
|
|
||||||
doc := &source.SourceDocument{
|
|
||||||
ID: "source-1",
|
|
||||||
Kind: "document",
|
|
||||||
Format: "text/plain",
|
|
||||||
Digest: "sha256:abc123",
|
|
||||||
Units: []source.SourceUnit{
|
|
||||||
{ID: 1, Kind: "section", Text: "First source text."},
|
|
||||||
{ID: 2, Kind: "section", Text: "Second source text."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
chunk := source.Chunk{
|
|
||||||
ID: "source-1:chunk:1",
|
|
||||||
SourceID: doc.ID,
|
|
||||||
Index: 1,
|
|
||||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 2},
|
|
||||||
Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`),
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: []source.SourceUnit{doc.Units[1]},
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{
|
|
||||||
Source: doc,
|
|
||||||
Chunk: &chunk,
|
|
||||||
AmbientContext: map[string]any{"mode": "chunked"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Extract() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if result.Value.Value != "chunked" {
|
|
||||||
t.Fatalf("Value = %q, want chunked", result.Value.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReferenceSetDataTypes(t *testing.T) {
|
|
||||||
references := ReferenceSet{
|
|
||||||
Slots: map[string]ResolvedReferenceSlot{
|
|
||||||
"roster": {
|
|
||||||
Slot: ReferenceSlot{
|
|
||||||
Name: "roster",
|
|
||||||
Description: "Known characters",
|
|
||||||
Required: true,
|
|
||||||
AcceptedMediaTypes: []string{"text/plain"},
|
|
||||||
Multiple: true,
|
|
||||||
MaxBytes: 4096,
|
|
||||||
},
|
|
||||||
Items: []ReferenceItem{
|
|
||||||
{
|
|
||||||
SlotName: "roster",
|
|
||||||
MediaType: "text/plain",
|
|
||||||
Content: []byte("Aria\nBryn\n"),
|
|
||||||
Digest: "sha256:reference",
|
|
||||||
Origin: ReferenceOrigin{
|
|
||||||
Type: "file",
|
|
||||||
URI: "file:///tmp/roster.txt",
|
|
||||||
},
|
|
||||||
SizeBytes: 10,
|
|
||||||
BindingSource: ReferenceBindingSourceConfig,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
item := references.Slots["roster"].Items[0]
|
|
||||||
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
|
|
||||||
t.Fatalf("reference item = %#v, want constructed item fields", item)
|
|
||||||
}
|
|
||||||
if item.BindingSource != ReferenceBindingSourceConfig {
|
|
||||||
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
|
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
|
||||||
if got := CloneReferenceSlots(nil); got != nil {
|
if got := CloneReferenceSlots(nil); got != nil {
|
||||||
t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got)
|
t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got)
|
||||||
@@ -319,61 +169,6 @@ func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
|
||||||
extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}}
|
|
||||||
merger := fakeMerger{key: "generic-merger"}
|
|
||||||
normalizer := fakeNormalizer{key: "generic-normalizer"}
|
|
||||||
encoder := fakeOutputEncoder{key: "generic-output"}
|
|
||||||
|
|
||||||
merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{
|
|
||||||
LaneID: "generic-lane",
|
|
||||||
ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Merge() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if merger.Key() != "generic-merger" {
|
|
||||||
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
|
|
||||||
}
|
|
||||||
if merged.Value.Value != "example" {
|
|
||||||
t.Fatalf("merged value = %q, want example", merged.Value.Value)
|
|
||||||
}
|
|
||||||
|
|
||||||
normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{
|
|
||||||
LaneID: "generic-lane",
|
|
||||||
MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Normalize() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if normalizer.Key() != "generic-normalizer" {
|
|
||||||
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
|
|
||||||
}
|
|
||||||
if normalized.Value.Value != "example" {
|
|
||||||
t.Fatalf("normalized value = %q, want example", normalized.Value.Value)
|
|
||||||
}
|
|
||||||
|
|
||||||
encoded, err := encoder.Encode(context.Background(), OutputRequest{
|
|
||||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
|
||||||
NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Encode() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if encoder.Key() != "generic-output" {
|
|
||||||
t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key())
|
|
||||||
}
|
|
||||||
if len(encoded.Files) != 1 {
|
|
||||||
t.Fatalf("len(Files) = %d, want 1", len(encoded.Files))
|
|
||||||
}
|
|
||||||
if encoded.Files[0].ContentType != "application/json" {
|
|
||||||
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
|
|
||||||
}
|
|
||||||
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
|
|
||||||
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
|
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
|
||||||
file := OutputFile{
|
file := OutputFile{
|
||||||
Name: "artifacts/events.json",
|
Name: "artifacts/events.json",
|
||||||
@@ -441,24 +236,6 @@ func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPla
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingChunker struct {
|
|
||||||
key string
|
|
||||||
request ChunkRequest
|
|
||||||
}
|
|
||||||
|
|
||||||
func (chunker *recordingChunker) Key() string {
|
|
||||||
return chunker.key
|
|
||||||
}
|
|
||||||
|
|
||||||
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
|
|
||||||
chunker.request = req
|
|
||||||
return fakeChunker{key: chunker.key}.Plan(ctx, req)
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeExtractor struct {
|
type fakeExtractor struct {
|
||||||
key string
|
key string
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,10 +56,6 @@ func TestMustLookupResponseSchemaPanicsForUnknownKey(t *testing.T) {
|
|||||||
|
|
||||||
func TestRegisteredResponseSchemasSortedByKey(t *testing.T) {
|
func TestRegisteredResponseSchemasSortedByKey(t *testing.T) {
|
||||||
schemas := RegisteredResponseSchemas()
|
schemas := RegisteredResponseSchemas()
|
||||||
if len(schemas) != 2 {
|
|
||||||
t.Fatalf("expected two schemas, got %d", len(schemas))
|
|
||||||
}
|
|
||||||
|
|
||||||
keys := make([]string, len(schemas))
|
keys := make([]string, len(schemas))
|
||||||
seen := make(map[ResponseSchemaKey]bool, len(schemas))
|
seen := make(map[ResponseSchemaKey]bool, len(schemas))
|
||||||
for i, schema := range schemas {
|
for i, schema := range schemas {
|
||||||
|
|||||||
@@ -1,175 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"testing/fstest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestScriptoriumPublicAPIGrounding(t *testing.T) {
|
|
||||||
// Keep this compile-time grounding close to the future Notarius adapter so
|
|
||||||
// dependency upgrades reveal API drift before the runtime cutover.
|
|
||||||
engine, err := scriptorium.NewEngine(
|
|
||||||
scriptorium.Config{
|
|
||||||
PromptDir: "unused-when-prompt-option-is-set",
|
|
||||||
ProfileDir: "",
|
|
||||||
SchemaDir: "",
|
|
||||||
Timeout: time.Second,
|
|
||||||
},
|
|
||||||
scriptorium.WithPromptFS(fstest.MapFS{}, "."),
|
|
||||||
scriptorium.WithProfileFS(fstest.MapFS{}, "."),
|
|
||||||
scriptorium.WithSchemaFS(fstest.MapFS{}, "."),
|
|
||||||
scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "test-profile",
|
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
|
||||||
Model: "test-model",
|
|
||||||
APIKeyRequired: true,
|
|
||||||
ExtraParams: map[string]any{"mode": "test"},
|
|
||||||
})),
|
|
||||||
scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if engine == nil {
|
|
||||||
t.Fatalf("NewEngine() = nil, want engine")
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
_ func(string) scriptorium.Option = scriptorium.WithPromptFile
|
|
||||||
_ func(string) scriptorium.Option = scriptorium.WithProfileFile
|
|
||||||
_ func(string) scriptorium.Option = scriptorium.WithSchemaFile
|
|
||||||
)
|
|
||||||
|
|
||||||
req := scriptorium.RunRequest{
|
|
||||||
PromptID: "dnd.spells",
|
|
||||||
PromptVersion: "v1",
|
|
||||||
ProfileID: "test-profile",
|
|
||||||
APIKey: "request-scoped-secret",
|
|
||||||
Inputs: map[string]scriptorium.ArtifactRef{
|
|
||||||
"transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`),
|
|
||||||
"glossary": scriptorium.Inline(""),
|
|
||||||
"roster": scriptorium.File("/tmp/roster.txt"),
|
|
||||||
},
|
|
||||||
Vars: map[string]string{
|
|
||||||
"session_id": "session-1",
|
|
||||||
},
|
|
||||||
Execution: &scriptorium.ExecutionTargetOverride{
|
|
||||||
Model: "override-model",
|
|
||||||
Temperature: ptr(0.2),
|
|
||||||
MaxTokens: ptr(100),
|
|
||||||
TopP: ptr(0.9),
|
|
||||||
TimeoutSeconds: ptr(30),
|
|
||||||
ServiceTier: "standard",
|
|
||||||
ReasoningEffort: "low",
|
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
|
||||||
ExtraParams: map[string]any{"provider_option": "value"},
|
|
||||||
},
|
|
||||||
Validation: &scriptorium.OutputContract{
|
|
||||||
Format: scriptorium.FormatJSON,
|
|
||||||
ValidationMode: scriptorium.ValidationJSONSchema,
|
|
||||||
SchemaPath: "schemas/dnd_spells.v1.json",
|
|
||||||
RepairAttempts: 1,
|
|
||||||
},
|
|
||||||
Metadata: map[string]string{
|
|
||||||
"artifact_kind": "dnd_spell",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline {
|
|
||||||
t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline)
|
|
||||||
}
|
|
||||||
if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile {
|
|
||||||
t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
result := scriptorium.RunResult{
|
|
||||||
RunID: "run-1",
|
|
||||||
Artifact: scriptorium.Artifact{
|
|
||||||
Name: "output",
|
|
||||||
ContentType: "application/json",
|
|
||||||
Body: []byte(`{"ok":true}`),
|
|
||||||
URI: "inline://output",
|
|
||||||
Size: int64(len(`{"ok":true}`)),
|
|
||||||
Hash: "sha256:abc",
|
|
||||||
},
|
|
||||||
RawOutput: `{"ok":true}`,
|
|
||||||
PromptID: req.PromptID,
|
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
PromptHash: "prompt-hash",
|
|
||||||
RenderedPromptHash: "rendered-prompt-hash",
|
|
||||||
SelectedProfileID: req.ProfileID,
|
|
||||||
ModelName: "test-model",
|
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{
|
|
||||||
Model: "test-model",
|
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
|
||||||
ExtraParams: map[string]any{"provider_option": "value"},
|
|
||||||
ReasoningEffort: "low",
|
|
||||||
},
|
|
||||||
InputHashes: map[string]string{
|
|
||||||
"transcript": "sha256:def",
|
|
||||||
},
|
|
||||||
Validation: scriptorium.ValidationResult{
|
|
||||||
Status: scriptorium.ValidationPassed,
|
|
||||||
Mode: scriptorium.ValidationJSONSchema,
|
|
||||||
SchemaPath: req.Validation.SchemaPath,
|
|
||||||
RepairAttempts: 1,
|
|
||||||
IsValid: true,
|
|
||||||
},
|
|
||||||
Usage: scriptorium.TokenUsage{
|
|
||||||
PromptTokens: 10,
|
|
||||||
CompletionTokens: 5,
|
|
||||||
TotalTokens: 15,
|
|
||||||
CachedTokens: 3,
|
|
||||||
CacheWriteTokens: 2,
|
|
||||||
},
|
|
||||||
StartTime: time.Unix(1, 0),
|
|
||||||
EndTime: time.Unix(2, 0),
|
|
||||||
Duration: time.Second,
|
|
||||||
}
|
|
||||||
if result.Validation.Status != scriptorium.ValidationPassed {
|
|
||||||
t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed)
|
|
||||||
}
|
|
||||||
if result.Usage.TotalTokens != 15 {
|
|
||||||
t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens)
|
|
||||||
}
|
|
||||||
|
|
||||||
publicErrors := []error{
|
|
||||||
scriptorium.ErrInvalidConfig,
|
|
||||||
scriptorium.ErrInvalidRequest,
|
|
||||||
scriptorium.ErrPromptNotFound,
|
|
||||||
scriptorium.ErrProfileNotFound,
|
|
||||||
scriptorium.ErrPromptLoad,
|
|
||||||
scriptorium.ErrProfileLoad,
|
|
||||||
scriptorium.ErrArtifactLoad,
|
|
||||||
scriptorium.ErrPromptRender,
|
|
||||||
scriptorium.ErrLLMGenerate,
|
|
||||||
scriptorium.ErrValidation,
|
|
||||||
}
|
|
||||||
for _, publicErr := range publicErrors {
|
|
||||||
if !errors.Is(publicErr, publicErr) {
|
|
||||||
t.Fatalf("sentinel error does not match itself: %v", publicErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type scriptoriumGroundingLLMClient struct{}
|
|
||||||
|
|
||||||
func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
|
||||||
return &scriptorium.GenerateResponse{
|
|
||||||
Content: `{"ok":true}`,
|
|
||||||
Usage: scriptorium.TokenUsage{
|
|
||||||
PromptTokens: 1,
|
|
||||||
CompletionTokens: 1,
|
|
||||||
TotalTokens: 2,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ptr[T any](v T) *T {
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
package pipeline_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
|
||||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPipelineConfigResolvesWithProductionDefaultsRegistered(t *testing.T) {
|
|
||||||
cfg := config.Default()
|
|
||||||
cfg.Pipelines = map[string]pipeline.PipelineProfile{
|
|
||||||
"defaults": {
|
|
||||||
Input: pipeline.Binding("input"),
|
|
||||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
|
||||||
"events": {Extract: pipeline.Binding("extract")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "defaults",
|
|
||||||
Catalog: defaultModuleCatalog(t),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
pipeline := resolved.ResolvedPipeline
|
|
||||||
if pipeline.Chunk.Module != units.Key {
|
|
||||||
t.Fatalf("Chunk.Module = %q, want %q", pipeline.Chunk.Module, units.Key)
|
|
||||||
}
|
|
||||||
if pipeline.Output.Module != jsonoutput.Key {
|
|
||||||
t.Fatalf("Output.Module = %q, want %q", pipeline.Output.Module, jsonoutput.Key)
|
|
||||||
}
|
|
||||||
lane := pipeline.ArtifactLanes[0]
|
|
||||||
if lane.Merge.Module != appendorder.Key {
|
|
||||||
t.Fatalf("Merge.Module = %q, want %q", lane.Merge.Module, appendorder.Key)
|
|
||||||
}
|
|
||||||
if lane.Normalize.Module != noop.Key {
|
|
||||||
t.Fatalf("Normalize.Module = %q, want %q", lane.Normalize.Module, noop.Key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
inputs := pipeline.NewInputAdapterRegistry()
|
|
||||||
chunkers := pipeline.NewChunkerRegistry()
|
|
||||||
extractors := pipeline.NewExtractorRegistry()
|
|
||||||
mergers := pipeline.NewMergerRegistry()
|
|
||||||
normalizers := pipeline.NewNormalizerRegistry()
|
|
||||||
outputs := pipeline.NewOutputEncoderRegistry()
|
|
||||||
|
|
||||||
if err := inputs.RegisterWithSpec(pipeline.ModuleSpec{
|
|
||||||
Key: "input",
|
|
||||||
Stage: pipeline.StageInput,
|
|
||||||
Provides: []string{"source"},
|
|
||||||
}, func() (contracts.InputAdapter, error) {
|
|
||||||
return defaultInput{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register input: %v", err)
|
|
||||||
}
|
|
||||||
if err := units.Register(chunkers); err != nil {
|
|
||||||
t.Fatalf("register generic chunker: %v", err)
|
|
||||||
}
|
|
||||||
if err := pipeline.RegisterExtractor[defaultArtifact](extractors, pipeline.ModuleSpec{
|
|
||||||
Key: "extract",
|
|
||||||
Stage: pipeline.StageExtract,
|
|
||||||
ArtifactKind: defaultArtifactKind,
|
|
||||||
Requires: []string{"chunks"},
|
|
||||||
Provides: []string{"records"},
|
|
||||||
}, func() (contracts.Extractor[defaultArtifact], error) {
|
|
||||||
return defaultExtractor{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register extractor: %v", err)
|
|
||||||
}
|
|
||||||
if err := appendorder.RegisterTyped(mergers, defaultArtifactKind, func(values []defaultArtifact) (defaultArtifact, error) {
|
|
||||||
if len(values) == 0 {
|
|
||||||
return defaultArtifact{}, nil
|
|
||||||
}
|
|
||||||
return values[0], nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register appendorder merger: %v", err)
|
|
||||||
}
|
|
||||||
if err := noop.RegisterTyped[defaultArtifact](normalizers, defaultArtifactKind); err != nil {
|
|
||||||
t.Fatalf("register noop normalizer: %v", err)
|
|
||||||
}
|
|
||||||
if err := jsonoutput.Register(outputs); err != nil {
|
|
||||||
t.Fatalf("register json output: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
codecs := pipeline.NewArtifactCodecRegistry()
|
|
||||||
if err := pipeline.RegisterArtifactCodec(codecs, defaultArtifactCodec{}); err != nil {
|
|
||||||
t.Fatalf("register artifact codec: %v", err)
|
|
||||||
}
|
|
||||||
return pipeline.ModuleCatalog{
|
|
||||||
Inputs: inputs,
|
|
||||||
Chunkers: chunkers,
|
|
||||||
ArtifactCodecs: codecs,
|
|
||||||
Extractors: extractors,
|
|
||||||
Mergers: mergers,
|
|
||||||
Normalizers: normalizers,
|
|
||||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
|
||||||
Outputs: outputs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type defaultInput struct{}
|
|
||||||
|
|
||||||
func (defaultInput) Key() string { return "input" }
|
|
||||||
|
|
||||||
func (defaultInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type defaultExtractor struct{}
|
|
||||||
|
|
||||||
func (defaultExtractor) Key() string { return "extract" }
|
|
||||||
|
|
||||||
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
||||||
|
|
||||||
func (defaultExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[defaultArtifact], error) {
|
|
||||||
return contracts.TypedExtractionResult[defaultArtifact]{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const defaultArtifactKind contracts.ArtifactKind = "test/default"
|
|
||||||
|
|
||||||
type defaultArtifact struct {
|
|
||||||
Value string `json:"value"`
|
|
||||||
}
|
|
||||||
type defaultArtifactCodec struct{}
|
|
||||||
|
|
||||||
func (defaultArtifactCodec) Kind() contracts.ArtifactKind { return defaultArtifactKind }
|
|
||||||
func (defaultArtifactCodec) Schema() contracts.ArtifactSchema {
|
|
||||||
return contracts.ArtifactSchema{ID: "urn:notarius:test:default", Name: "default", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}
|
|
||||||
}
|
|
||||||
func (defaultArtifactCodec) MediaType() string { return "application/json" }
|
|
||||||
func (defaultArtifactCodec) EncodeCandidate(value defaultArtifact) ([]byte, error) {
|
|
||||||
return json.Marshal(value)
|
|
||||||
}
|
|
||||||
func (defaultArtifactCodec) Encode(value defaultArtifact) ([]byte, error) { return json.Marshal(value) }
|
|
||||||
func (defaultArtifactCodec) Decode(content []byte) (defaultArtifact, error) {
|
|
||||||
var value defaultArtifact
|
|
||||||
err := json.Unmarshal(content, &value)
|
|
||||||
return value, err
|
|
||||||
}
|
|
||||||
@@ -2,8 +2,6 @@ package validate
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestApproved(t *testing.T) {
|
func TestApproved(t *testing.T) {
|
||||||
@@ -33,8 +31,3 @@ func TestRejectedTrimsReasonAndMessage(t *testing.T) {
|
|||||||
t.Fatalf("Message = %q, want message", result.Message)
|
t.Fatalf("Message = %q, want message", result.Message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHelpersReturnValidationResults(t *testing.T) {
|
|
||||||
var _ contracts.ValidationResult = Approved()
|
|
||||||
var _ contracts.ValidationResult = Rejected("reason", "message")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
package scenes
|
package scenes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadResponseSchemaForScenes(t *testing.T) {
|
func TestLoadResponseSchemaForScenes(t *testing.T) {
|
||||||
@@ -31,62 +35,83 @@ func TestLoadResponseSchemaForScenes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
|
func TestResponseSchemaValidatesSceneResponses(t *testing.T) {
|
||||||
schema, err := loadResponseSchema()
|
schema, err := loadResponseSchema()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var decoded map[string]any
|
valid := validSceneSchemaResponse()
|
||||||
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
|
validJSON, err := json.Marshal(valid)
|
||||||
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if decoded["$id"] != ResponseSchemaID {
|
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
|
||||||
t.Fatalf("$id = %#v, want %q", decoded["$id"], ResponseSchemaID)
|
t.Fatalf("valid scene response rejected: %v", err)
|
||||||
}
|
|
||||||
if decoded["additionalProperties"] != false {
|
|
||||||
t.Fatalf("additionalProperties = %#v, want false", decoded["additionalProperties"])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
properties := decoded["properties"].(map[string]any)
|
tests := []struct {
|
||||||
if _, ok := properties["artifact_type"]; ok {
|
name string
|
||||||
t.Fatal("schema includes artifact_type, want only scene response fields")
|
mutate func(map[string]any)
|
||||||
}
|
}{
|
||||||
if _, ok := properties["session_scope"]; ok {
|
{
|
||||||
t.Fatal("schema includes session_scope, want no session wrapper")
|
name: "obsolete segment boundaries",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
scene := response["scenes"].([]any)[0].(map[string]any)
|
||||||
|
scene["start_segment_id"] = 1
|
||||||
|
scene["end_segment_id"] = 2
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-positive start unit",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-positive end unit",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid primary mode",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["scenes"].([]any)[0].(map[string]any)["primary_mode"] = "Unknown"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid boundary confidence",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["scenes"].([]any)[0].(map[string]any)["boundary_confidence"] = "Unknown"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty boundary caveat",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["boundary_caveats"] = []any{""}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown property",
|
||||||
|
mutate: func(response map[string]any) {
|
||||||
|
response["unexpected"] = true
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
sceneProperties := properties["scenes"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
|
for _, tt := range tests {
|
||||||
for _, field := range []string{"scene_id", "start_segment_id", "end_segment_id"} {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
if _, ok := sceneProperties[field]; ok {
|
response := validSceneSchemaResponse()
|
||||||
t.Fatalf("scene schema includes old field %q", field)
|
tt.mutate(response)
|
||||||
|
content, err := json.Marshal(response)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||||
|
t.Fatal("validateJSONSchema() error = nil, want rejected response")
|
||||||
}
|
}
|
||||||
for _, field := range []string{"start_unit_id", "end_unit_id"} {
|
})
|
||||||
property := sceneProperties[field].(map[string]any)
|
|
||||||
if property["type"] != "integer" {
|
|
||||||
t.Fatalf("%s type = %#v, want integer", field, property["type"])
|
|
||||||
}
|
|
||||||
if property["minimum"] != float64(1) {
|
|
||||||
t.Fatalf("%s minimum = %#v, want 1", field, property["minimum"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
modeEnum := sceneProperties["primary_mode"].(map[string]any)["enum"].([]any)
|
|
||||||
if !sameStrings(modeEnum, []string{"Recap", "Discussion", "Combat", "Narrative"}) {
|
|
||||||
t.Fatalf("primary_mode enum = %#v, want Recap/Discussion/Combat/Narrative", modeEnum)
|
|
||||||
}
|
|
||||||
confidenceEnum := sceneProperties["boundary_confidence"].(map[string]any)["enum"].([]any)
|
|
||||||
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
|
||||||
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
|
||||||
}
|
|
||||||
|
|
||||||
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
|
|
||||||
if boundaryCaveatItems["type"] != "string" {
|
|
||||||
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
|
|
||||||
}
|
|
||||||
if boundaryCaveatItems["minLength"] != float64(1) {
|
|
||||||
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,14 +163,40 @@ func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func sameStrings(got []any, want []string) bool {
|
func validSceneSchemaResponse() map[string]any {
|
||||||
if len(got) != len(want) {
|
return map[string]any{
|
||||||
return false
|
"scenes": []any{
|
||||||
|
map[string]any{
|
||||||
|
"start_unit_id": 1,
|
||||||
|
"end_unit_id": 3,
|
||||||
|
"short_title": "Ambush",
|
||||||
|
"primary_mode": "Combat",
|
||||||
|
"main_participants": []any{"Aria"},
|
||||||
|
"summary": "The party fights.",
|
||||||
|
"boundary_note": "Combat starts and resolves.",
|
||||||
|
"boundary_confidence": "High",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"boundary_caveats": []any{},
|
||||||
}
|
}
|
||||||
for i := range want {
|
}
|
||||||
if got[i] != want[i] {
|
|
||||||
return false
|
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||||
}
|
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||||
}
|
if err != nil {
|
||||||
return true
|
return fmt.Errorf("parse instance: %w", err)
|
||||||
|
}
|
||||||
|
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse schema: %w", err)
|
||||||
|
}
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||||
|
return fmt.Errorf("load schema: %w", err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile("schema.json")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("compile schema: %w", err)
|
||||||
|
}
|
||||||
|
return schema.Validate(instance)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
|
func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
|
||||||
@@ -19,19 +21,23 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
|
|||||||
t.Fatalf("schema metadata = %#v, want valid hashed JSON", schema)
|
t.Fatalf("schema metadata = %#v, want valid hashed JSON", schema)
|
||||||
}
|
}
|
||||||
|
|
||||||
var decoded map[string]any
|
valid := validSpellsResponse()
|
||||||
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
|
validJSON, err := json.Marshal(valid)
|
||||||
t.Fatalf("Unmarshal(schema.JSONSchema) error = %v", err)
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(validSpellsResponse()) error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if decoded["$id"] != "notarius.dnd.spells.llm" {
|
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
|
||||||
t.Fatalf("LLM schema $id = %#v, want extractor transport schema", decoded["$id"])
|
t.Fatalf("valid private spells response rejected: %v", err)
|
||||||
}
|
}
|
||||||
properties := decoded["properties"].(map[string]any)
|
|
||||||
spellCastProperties := properties["spell_casts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
|
withCanonicalSourceID := validSpellsResponse()
|
||||||
sourceRefItems := spellCastProperties["source_refs"].(map[string]any)["items"].(map[string]any)
|
withCanonicalSourceID["spell_casts"].([]any)[0].(map[string]any)["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session-alpha"
|
||||||
sourceRefProperties := sourceRefItems["properties"].(map[string]any)
|
content, err := json.Marshal(withCanonicalSourceID)
|
||||||
if _, ok := sourceRefProperties["source_id"]; ok {
|
if err != nil {
|
||||||
t.Fatalf("LLM source ref schema contains canonical source_id: %#v", sourceRefProperties)
|
t.Fatalf("Marshal(response with source_id) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||||
|
t.Fatal("validateJSONSchema() error = nil, want canonical source_id rejected")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,3 +66,39 @@ func TestResponseSchemaDiagnosticsOmitRawSchema(t *testing.T) {
|
|||||||
t.Fatalf("diagnostics include raw schema: %#v", diagnostics)
|
t.Fatalf("diagnostics include raw schema: %#v", diagnostics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validSpellsResponse() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"spell_casts": []any{
|
||||||
|
map[string]any{
|
||||||
|
"caster": "Aria",
|
||||||
|
"spell": "Cure Wounds",
|
||||||
|
"effect": "The wounds close.",
|
||||||
|
"narrative_description": "Aria casts Cure Wounds.",
|
||||||
|
"source_refs": []any{
|
||||||
|
map[string]any{"start_unit_id": 1, "end_unit_id": 2},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||||
|
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile("schema.json")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return schema.Validate(instance)
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -19,12 +20,10 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
if err := Register(registries, assets); err != nil {
|
if err := Register(registries, assets); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"})
|
assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"})
|
||||||
assertKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
||||||
if got := registries.ArtifactCodecs.RegisteredKinds(); !reflect.DeepEqual(got, []contracts.ArtifactKind{"dnd/spell-list"}) {
|
assertContainsArtifactKinds(t, registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind})
|
||||||
t.Fatalf("artifact codec kinds = %#v, want dnd/spell-list", got)
|
assertContainsKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||||
}
|
|
||||||
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
|
||||||
"extract/dnd/spells/shape",
|
"extract/dnd/spells/shape",
|
||||||
"extract/dnd/spells/source_refs",
|
"extract/dnd/spells/source_refs",
|
||||||
"extract/dnd/spells/source_relatedness",
|
"extract/dnd/spells/source_relatedness",
|
||||||
@@ -41,7 +40,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
||||||
}
|
}
|
||||||
assertAssetNames(t, assets.PromptFS, []string{
|
assertAssetNamesContain(t, assets.PromptFS, []string{
|
||||||
"dnd.scenes/dnd.scenes.yaml",
|
"dnd.scenes/dnd.scenes.yaml",
|
||||||
"dnd.scenes/instructions.md",
|
"dnd.scenes/instructions.md",
|
||||||
"dnd.scenes/sharedassets/common-dnd-references.md",
|
"dnd.scenes/sharedassets/common-dnd-references.md",
|
||||||
@@ -55,10 +54,16 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
"dnd.spells/sharedassets/common-dnd-transcript.md",
|
"dnd.spells/sharedassets/common-dnd-transcript.md",
|
||||||
"dnd.spells/task.md",
|
"dnd.spells/task.md",
|
||||||
})
|
})
|
||||||
assertAssetNames(t, assets.SchemaFS, []string{
|
assertAssetNamesContain(t, assets.SchemaFS, []string{
|
||||||
"dnd_scenes.v1.json",
|
"dnd_scenes.v1.json",
|
||||||
"dnd_spells_llm.v1.json",
|
"dnd_spells_llm.v1.json",
|
||||||
})
|
})
|
||||||
|
if spec, ok := registries.Chunkers.Spec("dnd/scenes"); !ok || spec.Key != "dnd/scenes" {
|
||||||
|
t.Fatalf("scene chunker spec = %#v, present = %t; want family-owned spec", spec, ok)
|
||||||
|
}
|
||||||
|
if spec, ok := registries.Extractors.Spec(spells.Key); !ok || spec.ArtifactKind != dnd.SpellListKind {
|
||||||
|
t.Fatalf("spell extractor spec = %#v, present = %t; want dnd spell-list artifact", spec, ok)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
||||||
@@ -118,14 +123,33 @@ func completeRegistries() pipeline.Registries {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertKeys(t *testing.T, name string, got, want []string) {
|
func assertContainsKeys(t *testing.T, name string, got, want []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if !reflect.DeepEqual(got, want) {
|
seen := make(map[string]struct{}, len(got))
|
||||||
t.Fatalf("%s keys = %#v, want %#v", name, got, want)
|
for _, key := range got {
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, key := range want {
|
||||||
|
if _, ok := seen[key]; !ok {
|
||||||
|
t.Fatalf("%s keys = %#v, want required key %q", name, got, key)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) {
|
func assertContainsArtifactKinds(t *testing.T, got, want []contracts.ArtifactKind) {
|
||||||
|
t.Helper()
|
||||||
|
seen := make(map[contracts.ArtifactKind]struct{}, len(got))
|
||||||
|
for _, kind := range got {
|
||||||
|
seen[kind] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, kind := range want {
|
||||||
|
if _, ok := seen[kind]; !ok {
|
||||||
|
t.Fatalf("artifact codec kinds = %#v, want required kind %q", got, kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertAssetNamesContain(t *testing.T, getFS func() (fs.FS, error), want []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
fSys, err := getFS()
|
fSys, err := getFS()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -141,7 +165,13 @@ func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string)
|
|||||||
t.Fatalf("walk assets: %v", err)
|
t.Fatalf("walk assets: %v", err)
|
||||||
}
|
}
|
||||||
sort.Strings(got)
|
sort.Strings(got)
|
||||||
if !reflect.DeepEqual(got, want) {
|
seen := make(map[string]struct{}, len(got))
|
||||||
t.Fatalf("asset names = %#v, want %#v", got, want)
|
for _, name := range got {
|
||||||
|
seen[name] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, name := range want {
|
||||||
|
if _, ok := seen[name]; !ok {
|
||||||
|
t.Fatalf("asset names = %#v, want required asset %q", got, name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
package noop
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTypedNormalizerPreservesValue(t *testing.T) {
|
|
||||||
normalizer := NewTyped[string]()
|
|
||||||
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
|
|
||||||
if err != nil || result.Value != "value" {
|
|
||||||
t.Fatalf("result=%#v err=%v", result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package register
|
package register
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"reflect"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -13,21 +12,23 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
|
|||||||
if err := Register(registries, nil); err != nil {
|
if err := Register(registries, nil); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
|
assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
|
||||||
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), nil)
|
assertContainsKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||||
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), nil)
|
|
||||||
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
|
||||||
"generic/always_accept",
|
"generic/always_accept",
|
||||||
"generic/always_reject",
|
"generic/always_reject",
|
||||||
"generic/valid_json",
|
"generic/valid_json",
|
||||||
"generic/valid_json_schema",
|
"generic/valid_json_schema",
|
||||||
})
|
})
|
||||||
assertKeys(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
assertContainsKeys(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
||||||
if got := registries.Inputs.RegisteredKeys(); len(got) != 0 {
|
assertNoKeys(t, "inputs", registries.Inputs.RegisteredKeys(), "generic registrar to leave inputs unchanged")
|
||||||
t.Fatalf("input keys = %#v, want generic registrar to leave inputs unchanged", got)
|
assertNoKeys(t, "extractors", registries.Extractors.RegisteredKeys(), "generic registrar to leave extractors unchanged")
|
||||||
|
assertNoKeys(t, "mergers", registries.Mergers.RegisteredKeys(), "generic registrar to leave mergers for typed family composition")
|
||||||
|
assertNoKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), "generic registrar to leave normalizers for typed family composition")
|
||||||
|
if chunker, err := registries.Chunkers.Build("generic"); err != nil || chunker.Key() != "generic" {
|
||||||
|
t.Fatalf("build generic chunker = %v, %v; want generic implementation", chunker, err)
|
||||||
}
|
}
|
||||||
if got := registries.Extractors.RegisteredKeys(); len(got) != 0 {
|
if output, err := registries.Outputs.Build("json"); err != nil || output.Key() != "json" {
|
||||||
t.Fatalf("extractor keys = %#v, want generic registrar to leave extractors unchanged", got)
|
t.Fatalf("build json output = %v, %v; want json implementation", output, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,9 +84,22 @@ func completeRegistries() pipeline.Registries {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertKeys(t *testing.T, name string, got, want []string) {
|
func assertContainsKeys(t *testing.T, name string, got, want []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if !reflect.DeepEqual(got, want) {
|
seen := make(map[string]struct{}, len(got))
|
||||||
t.Fatalf("%s keys = %#v, want %#v", name, got, want)
|
for _, key := range got {
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, key := range want {
|
||||||
|
if _, ok := seen[key]; !ok {
|
||||||
|
t.Fatalf("%s keys = %#v, want required key %q", name, got, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoKeys(t *testing.T, name string, got []string, reason string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("%s keys = %#v, want %s", name, got, reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const moduleImportPrefix = "gitea.maximumdirect.net/eric/notarius/internal/modules/"
|
const moduleImportPrefix = "gitea.maximumdirect.net/eric/notarius/internal/modules/"
|
||||||
|
const internalImportPrefix = "gitea.maximumdirect.net/eric/notarius/internal/"
|
||||||
|
|
||||||
func TestProductionImportBoundaries(t *testing.T) {
|
func TestProductionImportBoundaries(t *testing.T) {
|
||||||
repositoryRoot := testRepositoryRoot(t)
|
repositoryRoot := testRepositoryRoot(t)
|
||||||
@@ -252,6 +253,39 @@ func TestImportBoundaryRules(t *testing.T) {
|
|||||||
importPath: moduleImportPrefix + "generic/chunk/units",
|
importPath: moduleImportPrefix + "generic/chunk/units",
|
||||||
wantError: true,
|
wantError: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "source production may import another core package",
|
||||||
|
filename: "internal/core/source/source.go",
|
||||||
|
sourcePackage: "source",
|
||||||
|
importPath: internalImportPrefix + "artifacts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source production cannot import framework",
|
||||||
|
filename: "internal/core/source/source.go",
|
||||||
|
sourcePackage: "source",
|
||||||
|
importPath: internalImportPrefix + "framework/contracts",
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source production cannot import modules",
|
||||||
|
filename: "internal/core/source/source.go",
|
||||||
|
sourcePackage: "source",
|
||||||
|
importPath: moduleImportPrefix + "dnd",
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "chunkplan production may import framework",
|
||||||
|
filename: "internal/framework/chunkplan/store.go",
|
||||||
|
sourcePackage: "chunkplan",
|
||||||
|
importPath: internalImportPrefix + "framework/contracts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "chunkplan production cannot import modules",
|
||||||
|
filename: "internal/framework/chunkplan/store.go",
|
||||||
|
sourcePackage: "chunkplan",
|
||||||
|
importPath: moduleImportPrefix + "dnd",
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "framework test may import module implementation",
|
name: "framework test may import module implementation",
|
||||||
filename: "internal/framework/pipeline/compatibility_test.go",
|
filename: "internal/framework/pipeline/compatibility_test.go",
|
||||||
@@ -345,6 +379,9 @@ func checkImportBoundaries(repositoryRoot string, filename string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateImport(filename string, sourcePackage string, importPath string) error {
|
func validateImport(filename string, sourcePackage string, importPath string) error {
|
||||||
|
if !strings.HasSuffix(filename, "_test.go") && strings.HasPrefix(filename, "internal/core/source/") && strings.HasPrefix(importPath, internalImportPrefix+"framework/") {
|
||||||
|
return importBoundaryViolation(filename, importPath, "core/source production code must not import framework or module implementations")
|
||||||
|
}
|
||||||
target, ok := moduleTargetForImport(importPath)
|
target, ok := moduleTargetForImport(importPath)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
@@ -384,6 +421,12 @@ func validateImport(filename string, sourcePackage string, importPath string) er
|
|||||||
}
|
}
|
||||||
return importBoundaryViolation(filename, importPath, "direct module imports from non-module tests are allowed only in CLI, core, and framework compatibility-test roots")
|
return importBoundaryViolation(filename, importPath, "direct module imports from non-module tests are allowed only in CLI, core, and framework compatibility-test roots")
|
||||||
}
|
}
|
||||||
|
if strings.HasPrefix(filename, "internal/core/source/") && strings.HasPrefix(importPath, moduleImportPrefix) {
|
||||||
|
return importBoundaryViolation(filename, importPath, "core/source production code must not import framework or module implementations")
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(filename, "internal/framework/chunkplan/") && strings.HasPrefix(importPath, moduleImportPrefix) {
|
||||||
|
return importBoundaryViolation(filename, importPath, "framework/chunkplan production code must not import module implementations")
|
||||||
|
}
|
||||||
if strings.HasPrefix(filename, "internal/framework/") || strings.HasPrefix(filename, "internal/core/") {
|
if strings.HasPrefix(filename, "internal/framework/") || strings.HasPrefix(filename, "internal/core/") {
|
||||||
return importBoundaryViolation(filename, importPath, "core and framework production code must not import module implementations")
|
return importBoundaryViolation(filename, importPath, "core and framework production code must not import module implementations")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,109 +18,102 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) {
|
func TestDNDSpellCapabilityFailures(t *testing.T) {
|
||||||
data, err := os.ReadFile("testdata/pipeline.yml")
|
tests := []struct {
|
||||||
if err != nil {
|
name string
|
||||||
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
mutate func(pipeline.ModuleSpec, pipeline.ModuleSpec) (pipeline.ModuleSpec, pipeline.ModuleSpec)
|
||||||
}
|
wantModule string
|
||||||
fileCfg, err := config.ParseFileConfigYAML(data)
|
wantCap string
|
||||||
if err != nil {
|
}{
|
||||||
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
{
|
||||||
|
name: "spell extractor requires transcript source",
|
||||||
|
mutate: func(input, extractor pipeline.ModuleSpec) (pipeline.ModuleSpec, pipeline.ModuleSpec) {
|
||||||
|
input.Provides = withoutCapability(input.Provides, "source.transcript")
|
||||||
|
return input, extractor
|
||||||
|
},
|
||||||
|
wantModule: spells.Key,
|
||||||
|
wantCap: "source.transcript",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "append-order merger requires spell casts",
|
||||||
|
mutate: func(input, extractor pipeline.ModuleSpec) (pipeline.ModuleSpec, pipeline.ModuleSpec) {
|
||||||
|
extractor.Provides = withoutCapability(extractor.Provides, "dnd.spell_casts")
|
||||||
|
return input, extractor
|
||||||
|
},
|
||||||
|
wantModule: pipeline.DefaultMergeModule,
|
||||||
|
wantCap: "dnd.spell_casts",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := config.Default()
|
for _, tt := range tests {
|
||||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
inputSpec, extractorSpec := tt.mutate(transcript.ModuleSpec(), spells.ModuleSpec())
|
||||||
|
_, err := pipeline.ResolvePipeline(dndCapabilityProfile(), pipeline.ResolveOptions{}, dndCapabilityCatalog(t, inputSpec, extractorSpec))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), tt.wantCap) || !strings.Contains(err.Error(), tt.wantModule) {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want %s missing %s capability", err, tt.wantModule, tt.wantCap)
|
||||||
}
|
}
|
||||||
|
|
||||||
resolved, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "dnd-spells-fixture",
|
|
||||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 {
|
|
||||||
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes))
|
|
||||||
}
|
|
||||||
lane := resolved.ResolvedPipeline.ArtifactLanes[0]
|
|
||||||
if lane.ID != "spells" {
|
|
||||||
t.Fatalf("lane ID = %q, want spells", lane.ID)
|
|
||||||
}
|
|
||||||
if lane.Extract.Module != spells.Key {
|
|
||||||
t.Fatalf("extract module = %q, want %q", lane.Extract.Module, spells.Key)
|
|
||||||
}
|
|
||||||
if resolved.ResolvedPipeline.Digest == "" {
|
|
||||||
t.Fatal("resolved digest is empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
again, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "dnd-spells-fixture",
|
|
||||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second Resolve() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
|
||||||
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) {
|
func dndCapabilityProfile() pipeline.PipelineProfile {
|
||||||
inputSpec := transcript.ModuleSpec()
|
return pipeline.PipelineProfile{
|
||||||
inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript")
|
ID: "dnd-capability",
|
||||||
chunkSpec := dndSpellsChunkerSpec()
|
Input: pipeline.Binding(transcript.Key),
|
||||||
chunkSpec.Requires = nil
|
Chunk: pipeline.Binding("fake/chunk"),
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
"spells": {Extract: pipeline.Binding(spells.Key)},
|
||||||
PipelineID: "dnd-spells-fixture",
|
},
|
||||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
|
||||||
input: inputSpec,
|
|
||||||
chunk: chunkSpec,
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "missing capability") ||
|
|
||||||
!strings.Contains(err.Error(), "source.transcript") ||
|
|
||||||
!strings.Contains(err.Error(), spells.Key) {
|
|
||||||
t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) {
|
func dndCapabilityCatalog(t *testing.T, inputSpec, extractorSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
|
||||||
extractorSpec := spells.ModuleSpec()
|
t.Helper()
|
||||||
extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts")
|
inputs := pipeline.NewInputAdapterRegistry()
|
||||||
|
if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) { return transcript.New(), nil }); err != nil {
|
||||||
|
t.Fatalf("register capability input: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
chunkers := pipeline.NewChunkerRegistry()
|
||||||
PipelineID: "dnd-spells-fixture",
|
if err := chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return dndSpellsChunker{}, nil }); err != nil {
|
||||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
t.Fatalf("register capability chunker: %v", err)
|
||||||
extractor: extractorSpec,
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "missing capability") ||
|
|
||||||
!strings.Contains(err.Error(), "dnd.spell_casts") ||
|
|
||||||
!strings.Contains(err.Error(), pipeline.DefaultMergeModule) {
|
|
||||||
t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
extractors := pipeline.NewExtractorRegistry()
|
||||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
extractorSpec.ArtifactKind = dnd.SpellListKind
|
||||||
PipelineID: "dnd-spells-fixture",
|
if err := pipeline.RegisterExtractor[dnd.SpellList](extractors, extractorSpec, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||||
Only: []string{"missing"},
|
return configExtractor{key: extractorSpec.Key}, nil
|
||||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
}); err != nil {
|
||||||
})
|
t.Fatalf("register capability extractor: %v", err)
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Resolve() error = nil, want unknown lane error")
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
|
||||||
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
codecs := pipeline.NewArtifactCodecRegistry()
|
||||||
|
if err := pipeline.RegisterArtifactCodec(codecs, spellcodec.New()); err != nil {
|
||||||
|
t.Fatalf("register capability codec: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mergers := pipeline.NewMergerRegistry()
|
||||||
|
if err := pipeline.RegisterMerger[dnd.SpellList](mergers, pipeline.ModuleSpec{
|
||||||
|
Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ArtifactKind: dnd.SpellListKind, Requires: []string{"dnd.spell_casts"},
|
||||||
|
}, func() (contracts.Merger[dnd.SpellList], error) { return appendorder.NewTyped(appendSpellLists) }); err != nil {
|
||||||
|
t.Fatalf("register capability merger: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizers := pipeline.NewNormalizerRegistry()
|
||||||
|
if err := pipeline.RegisterNormalizer[dnd.SpellList](normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ArtifactKind: dnd.SpellListKind}, func() (contracts.Normalizer[dnd.SpellList], error) {
|
||||||
|
return noop.NewTyped[dnd.SpellList](), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register capability normalizer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outputs := pipeline.NewOutputEncoderRegistry()
|
||||||
|
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput}, func() (contracts.OutputEncoder, error) { return dndSpellsOutput{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register capability output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return pipeline.ModuleCatalog{
|
||||||
|
Inputs: inputs, Chunkers: chunkers, ArtifactCodecs: codecs, Extractors: extractors,
|
||||||
|
Mergers: mergers, Normalizers: normalizers, ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -164,87 +164,6 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
|
|
||||||
raw := readDNDSpellsFixture(t)
|
|
||||||
resolved := resolveDNDSpellsPipeline(t)
|
|
||||||
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
|
|
||||||
"Mira: wizard who can cast Lightning Bolt",
|
|
||||||
"",
|
|
||||||
)
|
|
||||||
llmClient := &fakeSpellsLLMClient{
|
|
||||||
response: extractionResponse{SpellCasts: []spellCastResponse{}},
|
|
||||||
}
|
|
||||||
|
|
||||||
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
|
|
||||||
RawInput: raw,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(output.NormalizeOutputs) != 1 {
|
|
||||||
t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs))
|
|
||||||
}
|
|
||||||
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content)
|
|
||||||
if len(response.SpellCasts) != 0 {
|
|
||||||
t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts)
|
|
||||||
}
|
|
||||||
if len(llmClient.requests) != 1 {
|
|
||||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
|
||||||
}
|
|
||||||
request := llmClient.requests[0]
|
|
||||||
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
|
|
||||||
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
|
|
||||||
}
|
|
||||||
if got := string(request.Inputs["party"].Content); !strings.Contains(got, "Lightning Bolt") {
|
|
||||||
t.Fatalf("party input = %q, want party-reference-only spell in reference input", got)
|
|
||||||
}
|
|
||||||
if output.Manifest.ValidationStatus != "approved" {
|
|
||||||
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) {
|
|
||||||
raw := readDNDSpellsFixture(t)
|
|
||||||
resolved := resolveDNDSpellsPipeline(t)
|
|
||||||
llmClient := &fakeSpellsLLMClient{
|
|
||||||
response: extractionResponse{
|
|
||||||
SpellCasts: []spellCastResponse{
|
|
||||||
{
|
|
||||||
Caster: "Aria",
|
|
||||||
Spell: "Cure Wounds",
|
|
||||||
Effect: "Heals an injured ally.",
|
|
||||||
NarrativeDescription: "Aria restores the fighter after the fight.",
|
|
||||||
SourceRefs: responseSourceRefs("spell-session", 999, 999),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
|
|
||||||
RawInput: raw,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if len(output.NormalizeOutputs) != 1 {
|
|
||||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
|
||||||
}
|
|
||||||
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content)
|
|
||||||
if len(response.SpellCasts) != 1 {
|
|
||||||
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
|
|
||||||
}
|
|
||||||
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
|
||||||
t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
|
|
||||||
}
|
|
||||||
if len(output.Rejected) != 0 {
|
|
||||||
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
|
|
||||||
}
|
|
||||||
if output.Manifest.ValidationStatus != "approved" {
|
|
||||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
|
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
|
||||||
slots := make(map[string]contracts.ResolvedReferenceSlot)
|
slots := make(map[string]contracts.ResolvedReferenceSlot)
|
||||||
if strings.TrimSpace(party) != "" {
|
if strings.TrimSpace(party) != "" {
|
||||||
@@ -282,22 +201,6 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet
|
|||||||
return contracts.ReferenceSet{Slots: slots}
|
return contracts.ReferenceSet{Slots: slots}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) {
|
|
||||||
raw := readDNDSpellsFixture(t)
|
|
||||||
resolved := resolveDNDSpellsPipeline(t)
|
|
||||||
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
|
|
||||||
|
|
||||||
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
|
|
||||||
RawInput: raw,
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") {
|
|
||||||
t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err)
|
|
||||||
}
|
|
||||||
if len(output.NormalizeOutputs) != 0 {
|
|
||||||
t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
|
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -1,306 +0,0 @@
|
|||||||
package transcript
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPipelineConfigLoadsAndResolvesWithSeriatimInput(t *testing.T) {
|
|
||||||
cfg := loadPipelineConfig(t)
|
|
||||||
|
|
||||||
resolved, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "seriatim-fixture",
|
|
||||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resolved.ResolvedPipeline.Input.Module != Key {
|
|
||||||
t.Fatalf("resolved input module = %q, want %q", resolved.ResolvedPipeline.Input.Module, Key)
|
|
||||||
}
|
|
||||||
if resolved.ResolvedPipeline.Digest == "" {
|
|
||||||
t.Fatal("resolved digest is empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
again, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "seriatim-fixture",
|
|
||||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second Resolve() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
|
||||||
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPipelineConfigRejectsMissingSeriatimCapability(t *testing.T) {
|
|
||||||
spec := ModuleSpec()
|
|
||||||
spec.Provides = withoutCapability(spec.Provides, "transcript.timestamps")
|
|
||||||
cfg := loadPipelineConfig(t)
|
|
||||||
|
|
||||||
_, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "seriatim-fixture",
|
|
||||||
Catalog: seriatimTestCatalog(t, spec),
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), "transcript.timestamps") {
|
|
||||||
t.Fatalf("Resolve() error = %q, want missing transcript.timestamps capability", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
|
||||||
cfg := loadPipelineConfig(t)
|
|
||||||
|
|
||||||
_, err := cfg.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "seriatim-fixture",
|
|
||||||
Only: []string{"missing"},
|
|
||||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Resolve() error = nil, want unknown lane error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
|
||||||
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPipelineConfig(t *testing.T) config.Config {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
data, err := os.ReadFile("testdata/pipeline.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
fileCfg, err := config.ParseFileConfigYAML(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := config.Default()
|
|
||||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
|
||||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
profile, ok := cfg.Pipelines["seriatim-fixture"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("pipeline seriatim-fixture was not loaded")
|
|
||||||
}
|
|
||||||
if profile.Input.Module != Key {
|
|
||||||
t.Fatalf("loaded input module = %q, want %q", profile.Input.Module, Key)
|
|
||||||
}
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
inputs := pipeline.NewInputAdapterRegistry()
|
|
||||||
chunkers := pipeline.NewChunkerRegistry()
|
|
||||||
extractors := pipeline.NewExtractorRegistry()
|
|
||||||
mergers := pipeline.NewMergerRegistry()
|
|
||||||
normalizers := pipeline.NewNormalizerRegistry()
|
|
||||||
outputs := pipeline.NewOutputEncoderRegistry()
|
|
||||||
|
|
||||||
if reflect.DeepEqual(inputSpec, ModuleSpec()) {
|
|
||||||
if err := Register(inputs); err != nil {
|
|
||||||
t.Fatalf("register seriatim input: %v", err)
|
|
||||||
}
|
|
||||||
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
|
|
||||||
return New(), nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register seriatim input override: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{
|
|
||||||
Key: "fake/chunk",
|
|
||||||
Stage: pipeline.StageChunk,
|
|
||||||
Requires: []string{"source.transcript"},
|
|
||||||
Provides: []string{"chunks"},
|
|
||||||
})
|
|
||||||
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
|
|
||||||
Key: "fake/extract",
|
|
||||||
Stage: pipeline.StageExtract,
|
|
||||||
ArtifactKind: seriatimArtifactKind,
|
|
||||||
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
|
|
||||||
Provides: []string{"fake.artifacts"},
|
|
||||||
})
|
|
||||||
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
|
|
||||||
Key: pipeline.DefaultMergeModule,
|
|
||||||
Stage: pipeline.StageMerge,
|
|
||||||
ArtifactKind: seriatimArtifactKind,
|
|
||||||
Requires: []string{"fake.artifacts"},
|
|
||||||
})
|
|
||||||
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
|
|
||||||
Key: pipeline.DefaultNormalizeModule,
|
|
||||||
Stage: pipeline.StageNormalize,
|
|
||||||
ArtifactKind: seriatimArtifactKind,
|
|
||||||
})
|
|
||||||
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
|
|
||||||
Key: pipeline.DefaultOutputModule,
|
|
||||||
Stage: pipeline.StageOutput,
|
|
||||||
})
|
|
||||||
|
|
||||||
codecs := pipeline.NewArtifactCodecRegistry()
|
|
||||||
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
|
||||||
t.Fatalf("register artifact codec: %v", err)
|
|
||||||
}
|
|
||||||
return pipeline.ModuleCatalog{
|
|
||||||
Inputs: inputs,
|
|
||||||
Chunkers: chunkers,
|
|
||||||
ArtifactCodecs: codecs,
|
|
||||||
Extractors: extractors,
|
|
||||||
Mergers: mergers,
|
|
||||||
Normalizers: normalizers,
|
|
||||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
|
||||||
Outputs: outputs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
|
|
||||||
t.Helper()
|
|
||||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) {
|
|
||||||
return fakeChunker{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register chunker: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
|
||||||
t.Helper()
|
|
||||||
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) {
|
|
||||||
return fakeExtractor{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register extractor: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
|
||||||
t.Helper()
|
|
||||||
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) {
|
|
||||||
return fakeMerger{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register merger: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
|
||||||
t.Helper()
|
|
||||||
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) {
|
|
||||||
return fakeNormalizer{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register normalizer: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
|
|
||||||
t.Helper()
|
|
||||||
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) {
|
|
||||||
return fakeOutput{}, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("register output: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeChunker struct{}
|
|
||||||
|
|
||||||
func (fakeChunker) Key() string { return "fake/chunk" }
|
|
||||||
|
|
||||||
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
||||||
|
|
||||||
func (fakeChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
|
||||||
return contracts.ChunkPlanResult{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeExtractor struct{}
|
|
||||||
|
|
||||||
func (fakeExtractor) Key() string { return "fake/extract" }
|
|
||||||
|
|
||||||
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
||||||
|
|
||||||
func (fakeExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
|
||||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeMerger struct{}
|
|
||||||
|
|
||||||
func (fakeMerger) Key() string { return pipeline.DefaultMergeModule }
|
|
||||||
|
|
||||||
func (fakeMerger) Merge(ctx context.Context, req contracts.TypedMergeRequest[seriatimArtifact]) (contracts.TypedMergeResult[seriatimArtifact], error) {
|
|
||||||
if len(req.ExtractOutputs) == 0 {
|
|
||||||
return contracts.TypedMergeResult[seriatimArtifact]{}, nil
|
|
||||||
}
|
|
||||||
return contracts.TypedMergeResult[seriatimArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeNormalizer struct{}
|
|
||||||
|
|
||||||
func (fakeNormalizer) Key() string { return pipeline.DefaultNormalizeModule }
|
|
||||||
|
|
||||||
func (fakeNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
||||||
|
|
||||||
func (fakeNormalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[seriatimArtifact]) (contracts.TypedNormalizeResult[seriatimArtifact], error) {
|
|
||||||
return contracts.TypedNormalizeResult[seriatimArtifact]{Value: req.MergeOutput.Value}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeOutput struct{}
|
|
||||||
|
|
||||||
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
|
|
||||||
|
|
||||||
func (fakeOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
||||||
return contracts.OutputResult{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func withoutCapability(capabilities []string, capability string) []string {
|
|
||||||
filtered := make([]string, 0, len(capabilities))
|
|
||||||
for _, candidate := range capabilities {
|
|
||||||
if candidate != capability {
|
|
||||||
filtered = append(filtered, candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return filtered
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
_ contracts.Chunker = fakeChunker{}
|
|
||||||
_ contracts.Extractor[seriatimArtifact] = fakeExtractor{}
|
|
||||||
_ contracts.OutputEncoder = fakeOutput{}
|
|
||||||
)
|
|
||||||
|
|
||||||
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
|
|
||||||
|
|
||||||
type seriatimArtifact struct {
|
|
||||||
Value string `json:"value"`
|
|
||||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type seriatimArtifactCodec struct{}
|
|
||||||
|
|
||||||
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
|
|
||||||
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
|
|
||||||
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
|
||||||
}
|
|
||||||
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
|
|
||||||
func (seriatimArtifactCodec) EncodeCandidate(value seriatimArtifact) ([]byte, error) {
|
|
||||||
return json.Marshal(value)
|
|
||||||
}
|
|
||||||
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
|
|
||||||
return json.Marshal(value)
|
|
||||||
}
|
|
||||||
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
|
|
||||||
var value seriatimArtifact
|
|
||||||
err := json.Unmarshal(content, &value)
|
|
||||||
return value, err
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package transcript
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadPipelineConfig(t *testing.T) config.Config {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.Pipelines["seriatim-fixture"] = pipeline.PipelineProfile{
|
||||||
|
ID: "seriatim-fixture",
|
||||||
|
Input: pipeline.Binding(Key),
|
||||||
|
Chunk: pipeline.Binding("fake/chunk"),
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"events": {Extract: pipeline.Binding("fake/extract")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
inputs := pipeline.NewInputAdapterRegistry()
|
||||||
|
chunkers := pipeline.NewChunkerRegistry()
|
||||||
|
extractors := pipeline.NewExtractorRegistry()
|
||||||
|
mergers := pipeline.NewMergerRegistry()
|
||||||
|
normalizers := pipeline.NewNormalizerRegistry()
|
||||||
|
outputs := pipeline.NewOutputEncoderRegistry()
|
||||||
|
|
||||||
|
if reflect.DeepEqual(inputSpec, ModuleSpec()) {
|
||||||
|
if err := Register(inputs); err != nil {
|
||||||
|
t.Fatalf("register seriatim input: %v", err)
|
||||||
|
}
|
||||||
|
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
|
||||||
|
return New(), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register seriatim input override: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, Requires: []string{"source.transcript"}, Provides: []string{"chunks"}})
|
||||||
|
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract", Stage: pipeline.StageExtract, ArtifactKind: seriatimArtifactKind,
|
||||||
|
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"},
|
||||||
|
})
|
||||||
|
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ArtifactKind: seriatimArtifactKind, Requires: []string{"fake.artifacts"}})
|
||||||
|
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ArtifactKind: seriatimArtifactKind})
|
||||||
|
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput})
|
||||||
|
|
||||||
|
codecs := pipeline.NewArtifactCodecRegistry()
|
||||||
|
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
||||||
|
t.Fatalf("register artifact codec: %v", err)
|
||||||
|
}
|
||||||
|
return pipeline.ModuleCatalog{Inputs: inputs, Chunkers: chunkers, ArtifactCodecs: codecs, Extractors: extractors, Mergers: mergers, Normalizers: normalizers, ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: outputs}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return fakeChunker{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register chunker: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) { return fakeExtractor{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register extractor: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) { return fakeMerger{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register merger: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) { return fakeNormalizer{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register normalizer: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return fakeOutput{}, nil }); err != nil {
|
||||||
|
t.Fatalf("register output: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeChunker struct{}
|
||||||
|
|
||||||
|
func (fakeChunker) Key() string { return "fake/chunk" }
|
||||||
|
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
func (fakeChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
|
return contracts.ChunkPlanResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeExtractor struct{}
|
||||||
|
|
||||||
|
func (fakeExtractor) Key() string { return "fake/extract" }
|
||||||
|
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
func (fakeExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
||||||
|
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeMerger struct{}
|
||||||
|
|
||||||
|
func (fakeMerger) Key() string { return pipeline.DefaultMergeModule }
|
||||||
|
func (fakeMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[seriatimArtifact]) (contracts.TypedMergeResult[seriatimArtifact], error) {
|
||||||
|
if len(req.ExtractOutputs) == 0 {
|
||||||
|
return contracts.TypedMergeResult[seriatimArtifact]{}, nil
|
||||||
|
}
|
||||||
|
return contracts.TypedMergeResult[seriatimArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeNormalizer struct{}
|
||||||
|
|
||||||
|
func (fakeNormalizer) Key() string { return pipeline.DefaultNormalizeModule }
|
||||||
|
func (fakeNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
func (fakeNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[seriatimArtifact]) (contracts.TypedNormalizeResult[seriatimArtifact], error) {
|
||||||
|
return contracts.TypedNormalizeResult[seriatimArtifact]{Value: req.MergeOutput.Value}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeOutput struct{}
|
||||||
|
|
||||||
|
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
|
||||||
|
func (fakeOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||||
|
return contracts.OutputResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
|
||||||
|
|
||||||
|
type seriatimArtifact struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type seriatimArtifactCodec struct{}
|
||||||
|
|
||||||
|
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
|
||||||
|
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
|
||||||
|
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||||
|
}
|
||||||
|
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
|
||||||
|
func (seriatimArtifactCodec) EncodeCandidate(value seriatimArtifact) ([]byte, error) {
|
||||||
|
return json.Marshal(value)
|
||||||
|
}
|
||||||
|
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
|
||||||
|
return json.Marshal(value)
|
||||||
|
}
|
||||||
|
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
|
||||||
|
var value seriatimArtifact
|
||||||
|
err := json.Unmarshal(content, &value)
|
||||||
|
return value, err
|
||||||
|
}
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
version: 3
|
|
||||||
output:
|
|
||||||
directory: ./notarius-output
|
|
||||||
cache:
|
|
||||||
chunk_plans:
|
|
||||||
mode: bypass
|
|
||||||
checkpoints: {}
|
|
||||||
debug:
|
|
||||||
directory: ./notarius-debug
|
|
||||||
pipelines:
|
|
||||||
seriatim-fixture:
|
|
||||||
input: seriatim
|
|
||||||
chunk: fake/chunk
|
|
||||||
artifacts:
|
|
||||||
events:
|
|
||||||
extract: fake/extract
|
|
||||||
merge: appendorder
|
|
||||||
normalize: noop
|
|
||||||
output: json
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package register
|
package register
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"reflect"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -13,8 +12,22 @@ func TestRegisterAddsSeriatimFamily(t *testing.T) {
|
|||||||
if err := Register(registries, nil); err != nil {
|
if err := Register(registries, nil); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if got, want := registries.Inputs.RegisteredKeys(), []string{"seriatim"}; !reflect.DeepEqual(got, want) {
|
assertContainsKeys(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"})
|
||||||
t.Fatalf("input keys = %#v, want %#v", got, want)
|
if adapter, err := registries.Inputs.Build("seriatim"); err != nil || adapter.Key() != "seriatim" {
|
||||||
|
t.Fatalf("build seriatim input = %v, %v; want seriatim implementation", adapter, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertContainsKeys(t *testing.T, name string, got, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
seen := make(map[string]struct{}, len(got))
|
||||||
|
for _, key := range got {
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, key := range want {
|
||||||
|
if _, ok := seen[key]; !ok {
|
||||||
|
t.Fatalf("%s keys = %#v, want required key %q", name, got, key)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user