Compare commits
2 Commits
88042174b3
...
b4ee4c64f0
| Author | SHA1 | Date | |
|---|---|---|---|
| b4ee4c64f0 | |||
| 5a6e82f599 |
@@ -29,21 +29,25 @@ Source-format details belong in input modules. Transcript-specific concepts
|
|||||||
such as segments, speakers, timestamps, and transcript schemas must not spread
|
such as segments, speakers, timestamps, and transcript schemas must not spread
|
||||||
into runner, extractor, or validator framework code.
|
into runner, extractor, or validator framework code.
|
||||||
|
|
||||||
Extraction-domain details belong in process modules. D&D-specific concepts
|
Extraction-domain details belong in extract modules. D&D-specific concepts
|
||||||
such as spells, NPCs, items, combat turns, and encounters must not spread into
|
such as spells, NPCs, items, combat turns, and encounters must not spread into
|
||||||
core source, runner, or LLM framework packages.
|
core source, runner, or LLM framework packages.
|
||||||
|
|
||||||
Extracted facts should be grounded with source references. Source references
|
Extracted facts should be grounded with source references. Source references
|
||||||
should point to generic source units, not to transcript-only structures.
|
should point to generic source units, not to transcript-only structures.
|
||||||
|
Artifact records should require source references by default unless their schema
|
||||||
|
explicitly opts into ungrounded fields. Generic pipeline code should preserve
|
||||||
|
source-reference ranges exactly and should not merge or rewrite overlapping
|
||||||
|
ranges.
|
||||||
|
|
||||||
The application workflow is:
|
The application workflow is:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
These stages should remain explicit in the architecture. Chunking, merging, and
|
These stages should remain explicit in the architecture. Chunking, merging, and
|
||||||
normalization must not be hidden inside domain process modules when they represent
|
normalization must not be hidden inside domain extract modules when they represent
|
||||||
general pipeline behavior.
|
general pipeline behavior.
|
||||||
|
|
||||||
## Dependency Policy
|
## Dependency Policy
|
||||||
@@ -70,46 +74,38 @@ CLI and executable entrypoint:
|
|||||||
|
|
||||||
Core deterministic model and policy:
|
Core deterministic model and policy:
|
||||||
|
|
||||||
- `internal/core/config`: configuration structs, defaults, loading, precedence, and validation.
|
|
||||||
- `internal/core/source`: source document, source unit, and source reference types.
|
- `internal/core/source`: source document, source unit, and source reference types.
|
||||||
- `internal/core/sourcechunking`: deterministic chunking of ordered source units.
|
|
||||||
- `internal/core/artifacts`: artifact envelope, artifact candidates, rejected artifacts, and manifests.
|
- `internal/core/artifacts`: artifact envelope, artifact candidates, rejected artifacts, and manifests.
|
||||||
- `internal/core/diagnostics`: run directories and diagnostics artifact paths.
|
- `internal/core/config`: configuration structs, defaults, loading, precedence, and validation, once config exists.
|
||||||
- `internal/core/reporting`: process reports and report serialization.
|
|
||||||
- `internal/core/inputcatalog`: known input adapter keys and metadata.
|
|
||||||
- `internal/core/extractorcatalog`: known extractor keys and metadata.
|
|
||||||
|
|
||||||
Reusable framework plumbing:
|
Reusable framework plumbing:
|
||||||
|
|
||||||
- `internal/framework/contracts`: core interfaces and transport-neutral request/response contracts.
|
- `internal/framework/contracts`: core interfaces and transport-neutral request/response contracts.
|
||||||
- `internal/framework/runner`: orchestration across adapters, extractors, validators, and artifact output.
|
- `internal/framework/pipeline`: runner, pipeline-stage orchestration, registries, and small shared stage helpers.
|
||||||
- `internal/framework/pipeline`: shared pipeline-stage orchestration types, when needed.
|
- `internal/framework/validate`: shared validator runtime behavior and decision checks.
|
||||||
- `internal/framework/extraction`: shared extraction helper code.
|
- `internal/framework/llm`: LLM clients, scheduling, structured-output parsing, and response-schema registry, once LLM runtime exists.
|
||||||
- `internal/framework/merge`: shared merge-stage behavior.
|
- `internal/framework/prompt`: embedded prompt assets, prompt registry, and prompt rendering helpers, once prompt assets exist.
|
||||||
- `internal/framework/normalize`: shared normalization-stage behavior.
|
|
||||||
- `internal/framework/output`: output encoding contracts and shared helpers.
|
|
||||||
- `internal/framework/validators`: shared validator runtime behavior and decision checks.
|
|
||||||
- `internal/framework/llm`: LLM runtime, scheduling, and provider adapters.
|
|
||||||
- `internal/framework/responseschema`: embedded structured-output schema registry.
|
|
||||||
- `internal/framework/structuredoutput`: structured-output parsing and malformed-response handling.
|
|
||||||
- `internal/framework/promptcontext`: source-document prompt rendering helpers.
|
|
||||||
- `internal/framework/warnings`: shared warning records.
|
|
||||||
|
|
||||||
Domain implementations:
|
Domain implementations:
|
||||||
|
|
||||||
- `internal/modules/input/<name>`: input-stage modules that parse external input into core source documents.
|
- `internal/modules/input/<name>`: input-stage modules that parse external input into core source documents.
|
||||||
- `internal/modules/chunk/<name>`: chunk-stage modules.
|
- `internal/modules/chunk/<name>`: chunk-stage modules.
|
||||||
- `internal/modules/process/<domain>/<name>`: process-stage extractor modules.
|
- `internal/modules/extract/<domain>/<name>`: extract-stage extractor modules.
|
||||||
- `internal/modules/merge/<name>` or `internal/modules/merge/<domain>/<name>`: merge-stage modules.
|
- `internal/modules/merge/<name>` or `internal/modules/merge/<domain>/<name>`: merge-stage modules.
|
||||||
- `internal/modules/normalize/<name>` or `internal/modules/normalize/<domain>/<name>`: normalize-stage modules.
|
- `internal/modules/normalize/<name>` or `internal/modules/normalize/<domain>/<name>`: normalize-stage modules.
|
||||||
- `internal/modules/output/<name>`: output-stage modules.
|
- `internal/modules/output/<name>`: output-stage modules.
|
||||||
- `internal/validators/<validator>`: built-in validator implementations.
|
- `internal/validators/<validator>`: built-in validator implementations.
|
||||||
- `internal/prompts`: embedded prompt assets and prompt metadata registry.
|
|
||||||
- `internal/transport/http`: shared HTTP client code, if needed by provider integrations.
|
- `internal/transport/http`: shared HTTP client code, if needed by provider integrations.
|
||||||
|
|
||||||
Package-private implementation constants may live near the package that owns
|
Package-private implementation constants may live near the package that owns
|
||||||
them, preferably in `constants.go` when useful.
|
them, preferably in `constants.go` when useful.
|
||||||
|
|
||||||
|
Start with fewer, larger framework packages. Split a package only when a real
|
||||||
|
boundary proves itself through import direction, ownership, test seams, or
|
||||||
|
substantial file size. Do not create catalog, diagnostics, reporting,
|
||||||
|
structured-output, response-schema, output, merge, normalize, extraction, or
|
||||||
|
warnings packages merely because the concepts exist in the architecture.
|
||||||
|
|
||||||
## Stage Modules
|
## Stage Modules
|
||||||
|
|
||||||
Concrete business logic should live under `internal/modules/<stage>/...`.
|
Concrete business logic should live under `internal/modules/<stage>/...`.
|
||||||
@@ -120,7 +116,7 @@ workflow visible in the filesystem:
|
|||||||
```text
|
```text
|
||||||
internal/modules/input/...
|
internal/modules/input/...
|
||||||
internal/modules/chunk/...
|
internal/modules/chunk/...
|
||||||
internal/modules/process/...
|
internal/modules/extract/...
|
||||||
internal/modules/merge/...
|
internal/modules/merge/...
|
||||||
internal/modules/normalize/...
|
internal/modules/normalize/...
|
||||||
internal/modules/output/...
|
internal/modules/output/...
|
||||||
@@ -141,18 +137,22 @@ decisions.
|
|||||||
|
|
||||||
Other packages should interact with source input through adapter contracts and
|
Other packages should interact with source input through adapter contracts and
|
||||||
core source types. Input module implementation details and external dependency
|
core source types. Input module implementation details and external dependency
|
||||||
types must not leak into framework or process module packages.
|
types must not leak into framework or extract module packages.
|
||||||
|
|
||||||
Input module metadata may preserve source-specific facts such as transcript speaker,
|
Input module metadata may preserve source-specific facts such as transcript
|
||||||
timestamps, Markdown heading path, page number, or block ID. Framework code may
|
speaker, timestamps, Markdown heading path, page number, or block ID. Framework
|
||||||
carry metadata through, but should not require a specific adapter's metadata
|
code may carry metadata through, but should not require a specific adapter's
|
||||||
shape.
|
metadata shape.
|
||||||
|
|
||||||
|
Core source metadata should remain `map[string]any`. Document well-known keys
|
||||||
|
as conventions, and let input modules expose typed accessor helpers for their
|
||||||
|
own metadata when useful.
|
||||||
|
|
||||||
## Extractors
|
## Extractors
|
||||||
|
|
||||||
Extractors are independent modules that process source chunks or whole source
|
Extractors are independent modules that process source chunks or whole source
|
||||||
documents and produce one kind of structured artifact candidate.
|
documents and produce one kind of structured artifact candidate.
|
||||||
Each process module owns:
|
Each extract module owns:
|
||||||
|
|
||||||
- its artifact semantics;
|
- its artifact semantics;
|
||||||
- its prompt usage;
|
- its prompt usage;
|
||||||
@@ -160,9 +160,14 @@ Each process module owns:
|
|||||||
- its validator chain;
|
- its validator chain;
|
||||||
- any domain-specific mapping or interpretation.
|
- any domain-specific mapping or interpretation.
|
||||||
|
|
||||||
Process modules should depend on framework contracts and core source/artifact
|
Extract modules should depend on framework contracts and core source/artifact
|
||||||
types. They should not depend on concrete input module packages.
|
types. They should not depend on concrete input module packages.
|
||||||
|
|
||||||
|
Extraction requests should carry the active source chunk plus optional ambient
|
||||||
|
context, such as a document synopsis, prior-chunk summaries, known entities, or
|
||||||
|
other module-provided state. The context may be empty for simple modules, but
|
||||||
|
the contract should not assume extraction is always chunk-local.
|
||||||
|
|
||||||
Extractors should not be the only place where chunking, merging, or
|
Extractors should not be the only place where chunking, merging, or
|
||||||
normalization happens. They may choose processing mode or provide domain-specific
|
normalization happens. They may choose processing mode or provide domain-specific
|
||||||
merge/normalization behavior when generic behavior is insufficient, but the
|
merge/normalization behavior when generic behavior is insufficient, but the
|
||||||
@@ -181,7 +186,7 @@ The pipeline has six conceptual stages:
|
|||||||
|
|
||||||
1. input: external source material becomes a `SourceDocument`;
|
1. input: external source material becomes a `SourceDocument`;
|
||||||
2. chunk: a `SourceDocument` becomes ordered source chunks;
|
2. chunk: a `SourceDocument` becomes ordered source chunks;
|
||||||
3. process: extractors produce artifact candidates from chunks or whole documents;
|
3. extract: extractors produce artifact candidates from chunks or whole documents;
|
||||||
4. merge: per-chunk candidates become a merged candidate collection;
|
4. merge: per-chunk candidates become a merged candidate collection;
|
||||||
5. normalize: merged candidates are reconciled for duplicates, aliases, consistency, or cross-chunk issues;
|
5. normalize: merged candidates are reconciled for duplicates, aliases, consistency, or cross-chunk issues;
|
||||||
6. output: final artifacts are serialized.
|
6. output: final artifacts are serialized.
|
||||||
@@ -200,6 +205,15 @@ The framework should allow serial and parallel chunk processing. The first
|
|||||||
implementation may execute chunks serially for determinism, but contracts should
|
implementation may execute chunks serially for determinism, but contracts should
|
||||||
not prevent later parallel execution.
|
not prevent later parallel execution.
|
||||||
|
|
||||||
|
Final durable output should use one artifact file per artifact type plus a
|
||||||
|
run-level manifest/index file. Framework artifact flow should use a generic
|
||||||
|
envelope with `json.RawMessage` payloads; extract modules should own typed Go
|
||||||
|
structs at their boundaries and encode into that generic envelope before
|
||||||
|
returning to framework code.
|
||||||
|
|
||||||
|
Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||||
|
format version.
|
||||||
|
|
||||||
## Validators
|
## Validators
|
||||||
|
|
||||||
Validators should be independently testable and composable.
|
Validators should be independently testable and composable.
|
||||||
@@ -209,7 +223,11 @@ present. Validator decision semantics should be explicit: each candidate
|
|||||||
artifact should receive exactly one decision from each validator that evaluates
|
artifact should receive exactly one decision from each validator that evaluates
|
||||||
it.
|
it.
|
||||||
|
|
||||||
Shared validator runtime mechanics belong under `internal/framework/validators`.
|
LLM-backed review belongs in module-owned validator chains, not in a separate
|
||||||
|
global review phase. Extract modules and normalize modules may both use
|
||||||
|
deterministic and LLM-backed validators.
|
||||||
|
|
||||||
|
Shared validator runtime mechanics belong under `internal/framework/validate`.
|
||||||
Concrete validator behavior belongs under `internal/validators/<validator>`.
|
Concrete validator behavior belongs under `internal/validators/<validator>`.
|
||||||
|
|
||||||
## LLM Runtime
|
## LLM Runtime
|
||||||
@@ -290,11 +308,17 @@ be opt-in.
|
|||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Core logic should be testable without real external services. Use fakes,
|
Core logic should be testable without real external services. Use fakes,
|
||||||
fixtures, or local test doubles for adapters, extractors, validators, and LLM
|
fixtures, or local test doubles for input modules, extract modules, validators,
|
||||||
clients where practical.
|
and LLM clients where practical.
|
||||||
|
|
||||||
Contract-first work should include fake implementations that prove interfaces
|
Contract-first work should include fake implementations that prove interfaces
|
||||||
compose before real adapters or extractors depend on them.
|
compose before real input modules or extract modules depend on them.
|
||||||
|
|
||||||
|
Once the pipeline-stage contracts exist, maintain a fixture-driven walking
|
||||||
|
skeleton that exercises input, chunk, extract, merge, normalize, and output
|
||||||
|
stages with fake modules and fake external clients. This test should protect
|
||||||
|
stage composition continuously while real modules are introduced over later
|
||||||
|
checkpoints.
|
||||||
|
|
||||||
Config examples should be load-tested once config files exist. Important CLI
|
Config examples should be load-tested once config files exist. Important CLI
|
||||||
workflows should have parser or command tests. Adapter, extractor, and validator
|
workflows should have parser or command tests. Adapter, extractor, and validator
|
||||||
@@ -312,7 +336,7 @@ unit, source reference, input adapter, extractor, chunker, merger, normalizer,
|
|||||||
artifact, validator, and run manifest.
|
artifact, validator, and run manifest.
|
||||||
|
|
||||||
Source-format details belong in input module or integration docs.
|
Source-format details belong in input module or integration docs.
|
||||||
Domain-specific extraction details belong in process module or artifact docs.
|
Domain-specific extraction details belong in extract module or artifact docs.
|
||||||
|
|
||||||
When changing architecture, config, CLI behavior, stage modules, extractor
|
When changing architecture, config, CLI behavior, stage modules, extractor
|
||||||
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
|
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ In scope:
|
|||||||
Out of scope:
|
Out of scope:
|
||||||
|
|
||||||
- real input modules;
|
- real input modules;
|
||||||
- real process modules;
|
- real extract modules;
|
||||||
- LLM provider calls;
|
- LLM provider calls;
|
||||||
- prompt or response-schema assets;
|
- prompt or response-schema assets;
|
||||||
- diagnostics run directory;
|
- diagnostics run directory;
|
||||||
@@ -47,7 +47,7 @@ contract packages:
|
|||||||
structured LLM interfaces used by later checkpoints.
|
structured LLM interfaces used by later checkpoints.
|
||||||
|
|
||||||
The contracts should be proven with fake implementations in tests. Those tests
|
The contracts should be proven with fake implementations in tests. Those tests
|
||||||
should demonstrate composition without real input modules, real process modules,
|
should demonstrate composition without real input modules, real extract modules,
|
||||||
LLM provider calls, prompt assets, or diagnostics infrastructure.
|
LLM provider calls, prompt assets, or diagnostics infrastructure.
|
||||||
|
|
||||||
Implementation staging belongs in
|
Implementation staging belongs in
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ Out of scope:
|
|||||||
|
|
||||||
- real input parsing;
|
- real input parsing;
|
||||||
- real input modules;
|
- real input modules;
|
||||||
- real process modules;
|
- real extract modules;
|
||||||
- real LLM calls;
|
- real LLM calls;
|
||||||
- prompt assets;
|
- prompt assets;
|
||||||
- response schema assets;
|
- response schema assets;
|
||||||
@@ -38,13 +38,11 @@ Out of scope:
|
|||||||
|
|
||||||
The repository should contain a minimal framework composition layer:
|
The repository should contain a minimal framework composition layer:
|
||||||
|
|
||||||
- `internal/framework/inputregistry` registers and builds input adapter
|
- `internal/framework/pipeline` registers and builds input adapter constructors
|
||||||
constructors by stable key.
|
and extractor constructors by stable key.
|
||||||
- `internal/framework/extractorregistry` registers and builds extractor
|
- `internal/framework/validate` provides shared validator decision helpers and
|
||||||
constructors by stable key.
|
|
||||||
- `internal/framework/validators` provides shared validator decision helpers and
|
|
||||||
cardinality checks.
|
cardinality checks.
|
||||||
- `internal/framework/runner` executes configured extractors against a
|
- `internal/framework/pipeline` executes configured extractors against a
|
||||||
`SourceDocument`, applies validator chains, and returns approved and rejected
|
`SourceDocument`, applies validator chains, and returns approved and rejected
|
||||||
artifacts.
|
artifacts.
|
||||||
|
|
||||||
@@ -60,7 +58,7 @@ Implementation staging belongs in
|
|||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Fake adapter/extractor/validator registrations work in tests.
|
- Fake adapter/extractor/validator registrations work in tests.
|
||||||
- The runner operates on `SourceDocument`, not transcript-specific structures.
|
- The runner operates on `SourceDocument`, not transcript-specific structures.
|
||||||
- The runner does not import concrete D&D process module packages.
|
- The runner does not import concrete D&D extract module packages.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
|
|||||||
@@ -7,18 +7,20 @@ This document describes planned work, not implemented behavior.
|
|||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Make Notarius's application workflow first-class before adding real input
|
Make Notarius's application workflow first-class before adding real input
|
||||||
modules or process modules.
|
modules or extract modules.
|
||||||
|
|
||||||
The workflow should be:
|
The workflow should be:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
Checkpoint 3 should define the contracts and minimal fake-tested framework
|
Checkpoint 3 should define the contracts and minimal fake-tested framework
|
||||||
behavior for chunking, per-chunk processing, merging, and normalization. It
|
behavior for chunking, per-chunk extraction, merging, and normalization. It
|
||||||
should not add real input modules, real domain process modules, LLM provider code,
|
should also introduce a fixture-driven walking skeleton that exercises the full
|
||||||
or output encoders.
|
stage sequence with fake modules and a fake LLM client. It should not add real
|
||||||
|
input modules, real domain extract modules, LLM provider code, or production
|
||||||
|
output modules.
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
@@ -26,13 +28,16 @@ In scope:
|
|||||||
|
|
||||||
- source chunk model;
|
- source chunk model;
|
||||||
- chunker contract;
|
- chunker contract;
|
||||||
- process-stage contract for extractors operating on chunks;
|
- extract-stage contract for extractors operating on chunks;
|
||||||
- merge-stage contract;
|
- merge-stage contract;
|
||||||
- normalize-stage contract;
|
- normalize-stage contract;
|
||||||
- output-stage contract shape, if useful for pipeline completeness;
|
- output-stage contract and fake output encoder for pipeline completeness;
|
||||||
- runner/pipeline updates that exercise these stages with fake components;
|
- runner/pipeline updates that exercise these stages with fake components;
|
||||||
- generic append/chronological merge behavior for artifact candidates when
|
- generic append/chronological merge behavior for artifact candidates when
|
||||||
appropriate.
|
appropriate;
|
||||||
|
- fixture-driven walking skeleton test for
|
||||||
|
`input -> chunk -> extract -> merge -> normalize -> output`;
|
||||||
|
- fake `StructuredLLMClient` wired through a trivial extractor.
|
||||||
|
|
||||||
Out of scope:
|
Out of scope:
|
||||||
|
|
||||||
@@ -42,7 +47,8 @@ Out of scope:
|
|||||||
- prompt assets;
|
- prompt assets;
|
||||||
- response schema assets;
|
- response schema assets;
|
||||||
- diagnostics run directory;
|
- diagnostics run directory;
|
||||||
- production output serialization.
|
- real CLI command behavior;
|
||||||
|
- production output serialization or durable output writing.
|
||||||
|
|
||||||
## Target End State
|
## Target End State
|
||||||
|
|
||||||
@@ -50,15 +56,26 @@ The repository should contain explicit pipeline-stage contracts:
|
|||||||
|
|
||||||
- `InputAdapter`: external source input to `SourceDocument`.
|
- `InputAdapter`: external source input to `SourceDocument`.
|
||||||
- `Chunker`: `SourceDocument` to ordered `SourceChunk` values.
|
- `Chunker`: `SourceDocument` to ordered `SourceChunk` values.
|
||||||
- `Extractor` or processor: `SourceChunk` to artifact candidates.
|
- `Extractor`: `SourceChunk` to artifact candidates.
|
||||||
- `Merger`: per-chunk candidates to merged candidates.
|
- `Merger`: per-chunk candidates to merged candidates.
|
||||||
- `Normalizer`: merged candidates to normalized candidates.
|
- `Normalizer`: merged candidates to normalized candidates.
|
||||||
- `OutputEncoder`: final artifact bundle to bytes, if introduced in this
|
- `OutputEncoder`: final artifact bundle to bytes.
|
||||||
checkpoint.
|
|
||||||
|
|
||||||
The runner should orchestrate fake implementations through chunk, process,
|
The runner should orchestrate fake implementations through chunk, extract,
|
||||||
merge, normalize, and approval/validation behavior in tests.
|
merge, normalize, and approval/validation behavior in tests.
|
||||||
|
|
||||||
|
The checkpoint should include one fixture-driven integration test that starts
|
||||||
|
from fixture input bytes and ends at encoded output bytes. The fixture should
|
||||||
|
use a fake input adapter, deterministic chunker, trivial extractor, fake
|
||||||
|
structured LLM client, generic merger, no-op normalizer, and fake or minimal
|
||||||
|
JSON output encoder. This is a contract exercise, not a useful user-facing
|
||||||
|
workflow.
|
||||||
|
|
||||||
|
The walking skeleton should live in `internal/framework/pipeline`, with
|
||||||
|
fixtures under that package's `testdata/`. If the CLI has an extract command by
|
||||||
|
then, the same fixture may also be exercised through CLI wiring; if not, CLI
|
||||||
|
coverage remains deferred.
|
||||||
|
|
||||||
## Design Intent
|
## Design Intent
|
||||||
|
|
||||||
Chunking is a core application concern because many source documents, especially
|
Chunking is a core application concern because many source documents, especially
|
||||||
@@ -109,21 +126,43 @@ Domain-specific normalizers may later:
|
|||||||
- enforce chronological or source-reference consistency;
|
- enforce chronological or source-reference consistency;
|
||||||
- attach normalization warnings.
|
- attach normalization warnings.
|
||||||
|
|
||||||
|
## Walking Skeleton Fixture
|
||||||
|
|
||||||
|
The fixture-driven skeleton should prove the staged architecture continuously as
|
||||||
|
new contracts are added. It should be deliberately small:
|
||||||
|
|
||||||
|
- a fixture source document or raw source input with two or three source units;
|
||||||
|
- a fake input adapter that parses the fixture into `SourceDocument`;
|
||||||
|
- a deterministic chunker that produces multiple chunks;
|
||||||
|
- a trivial extractor that calls the fake structured LLM client and emits one
|
||||||
|
artifact candidate per chunk;
|
||||||
|
- a generic append-in-chunk-order merger;
|
||||||
|
- a no-op normalizer;
|
||||||
|
- a fake or minimal JSON output encoder;
|
||||||
|
- an expected output fixture checked byte-for-byte or by unmarshaled structural
|
||||||
|
equality.
|
||||||
|
|
||||||
|
The fake LLM client should be part of the test setup so the contract is
|
||||||
|
exercised without introducing provider code, prompt assets, or response schema
|
||||||
|
assets.
|
||||||
|
|
||||||
## Done Criteria
|
## Done Criteria
|
||||||
|
|
||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Pipeline-stage contracts are explicit and source/domain agnostic.
|
- Pipeline-stage contracts are explicit and source/domain agnostic.
|
||||||
- Fake tests prove input source documents can be chunked, processed, merged, and
|
- Fake tests prove input source documents can be chunked, extracted, merged, and
|
||||||
normalized.
|
normalized.
|
||||||
|
- A fixture-driven walking skeleton proves fake input, chunk, extract, merge,
|
||||||
|
normalize, and output modules compose end to end with a fake LLM client.
|
||||||
- Merge and normalize are distinct concepts in code and tests.
|
- Merge and normalize are distinct concepts in code and tests.
|
||||||
- The runner no longer implies whole-document-only extraction as the core
|
- The runner no longer implies whole-document-only extraction as the core
|
||||||
application model.
|
application model.
|
||||||
- No concrete input module, process module, LLM provider, prompt, response schema,
|
- No concrete input module, domain extract module, LLM provider, prompt,
|
||||||
diagnostics, config, or D&D artifact code is added.
|
response schema, diagnostics, config, or D&D artifact code is added.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
- Is the workflow clearly represented as input, chunk, process, merge,
|
- Is the workflow clearly represented as input, chunk, extract, merge,
|
||||||
normalize, and output?
|
normalize, and output?
|
||||||
- Are merge and normalize cleanly separated?
|
- Are merge and normalize cleanly separated?
|
||||||
- Can a generic merger handle simple chronological artifact streams?
|
- Can a generic merger handle simple chronological artifact streams?
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ This document describes planned work, not implemented behavior.
|
|||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Implement the first useful process-stage module: D&D spell casts from a
|
Implement the first useful extract-stage module: D&D spell casts from a
|
||||||
Seriatim transcript source document.
|
Seriatim transcript source document.
|
||||||
|
|
||||||
This checkpoint should produce the first meaningful vertical slice from real
|
This checkpoint should produce the first meaningful vertical slice from real
|
||||||
@@ -19,7 +19,7 @@ In scope:
|
|||||||
- D&D spell artifact schema and Go structs;
|
- D&D spell artifact schema and Go structs;
|
||||||
- structured response schema asset;
|
- structured response schema asset;
|
||||||
- prompt assets;
|
- prompt assets;
|
||||||
- `internal/modules/process/dnd/spells`;
|
- `internal/modules/extract/dnd/spells`;
|
||||||
- source-reference and schema validators in the extractor chain;
|
- source-reference and schema validators in the extractor chain;
|
||||||
- fake LLM tests;
|
- fake LLM tests;
|
||||||
- CLI-level integration test if the CLI path is ready.
|
- CLI-level integration test if the CLI path is ready.
|
||||||
@@ -50,7 +50,7 @@ type SpellCast struct {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Keep this schema inside the D&D spells process module or a D&D artifact package,
|
Keep this schema inside the D&D spells extract module or a D&D artifact package,
|
||||||
not inside core framework packages.
|
not inside core framework packages.
|
||||||
|
|
||||||
### Stage 2: Structured Response Schema
|
### Stage 2: Structured Response Schema
|
||||||
@@ -76,7 +76,7 @@ Prompts should:
|
|||||||
|
|
||||||
### Stage 4: Process Module Implementation
|
### Stage 4: Process Module Implementation
|
||||||
|
|
||||||
Implement `internal/modules/process/dnd/spells`.
|
Implement `internal/modules/extract/dnd/spells`.
|
||||||
|
|
||||||
The extractor should:
|
The extractor should:
|
||||||
|
|
||||||
@@ -117,13 +117,13 @@ The test should use fake LLM wiring and fixture input.
|
|||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Seriatim input can flow through the runner into the D&D spells extractor.
|
- Seriatim input can flow through the runner into the D&D spells extractor.
|
||||||
- Spell artifacts include valid source references.
|
- Spell artifacts include valid source references.
|
||||||
- D&D concepts are contained in process module/artifact packages and docs.
|
- D&D concepts are contained in extract module/artifact packages and docs.
|
||||||
- The first meaningful vertical slice is available through tests, and through
|
- The first meaningful vertical slice is available through tests, and through
|
||||||
CLI if the CLI path is ready.
|
CLI if the CLI path is ready.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
- Is the spell process module domain-specific without making the framework
|
- Is the spell extract module domain-specific without making the framework
|
||||||
D&D-specific?
|
D&D-specific?
|
||||||
- Are source references valid and useful for downstream validation?
|
- Are source references valid and useful for downstream validation?
|
||||||
- Is prompt/schema ownership clear?
|
- Is prompt/schema ownership clear?
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ not describe implemented behavior.
|
|||||||
Notarius documentation should make three boundaries obvious:
|
Notarius documentation should make three boundaries obvious:
|
||||||
|
|
||||||
- source-format support belongs to input-stage modules;
|
- source-format support belongs to input-stage modules;
|
||||||
- extraction-domain behavior belongs to process-stage modules;
|
- extraction-domain behavior belongs to extract-stage modules;
|
||||||
- core framework behavior is source-agnostic and domain-agnostic.
|
- core framework behavior is source-agnostic and domain-agnostic.
|
||||||
|
|
||||||
Documentation should avoid making the MVP look more transcript-specific or
|
Documentation should avoid making the MVP look more transcript-specific or
|
||||||
@@ -47,7 +47,7 @@ Core docs should avoid transcript-specific terms such as segment, speaker,
|
|||||||
timestamp, and transcript range unless discussing an input adapter or an example.
|
timestamp, and transcript range unless discussing an input adapter or an example.
|
||||||
|
|
||||||
Core docs should avoid D&D-specific terms such as spell, NPC, item, combat, and
|
Core docs should avoid D&D-specific terms such as spell, NPC, item, combat, and
|
||||||
encounter unless discussing process modules, artifact docs, or examples.
|
encounter unless discussing extract modules, artifact docs, or examples.
|
||||||
|
|
||||||
### Input Module Docs Own Source Formats
|
### Input Module Docs Own Source Formats
|
||||||
|
|
||||||
@@ -95,12 +95,12 @@ Stage module docs should cover:
|
|||||||
- prompt and response-schema ownership;
|
- prompt and response-schema ownership;
|
||||||
- examples.
|
- examples.
|
||||||
|
|
||||||
D&D concepts should be documented in D&D process-module or artifact docs, not in
|
D&D concepts should be documented in D&D extract-module or artifact docs, not in
|
||||||
generic runner or framework docs.
|
generic runner or framework docs.
|
||||||
|
|
||||||
### CLI Docs Should Reflect Extensibility
|
### CLI Docs Should Reflect Extensibility
|
||||||
|
|
||||||
The CLI reference should present input modules, chunk modules, process modules,
|
The CLI reference should present input modules, chunk modules, extract modules,
|
||||||
merge modules, normalize modules, and output modules as selectable or
|
merge modules, normalize modules, and output modules as selectable or
|
||||||
configurable components as they become user-facing.
|
configurable components as they become user-facing.
|
||||||
|
|
||||||
@@ -114,7 +114,7 @@ Once implemented, `docs/cli.md` should document:
|
|||||||
|
|
||||||
- positional source input path;
|
- positional source input path;
|
||||||
- input module selection;
|
- input module selection;
|
||||||
- process module selection;
|
- extract module selection;
|
||||||
- chunk/merge/normalize/output selection when configurable;
|
- chunk/merge/normalize/output selection when configurable;
|
||||||
- config path behavior;
|
- config path behavior;
|
||||||
- output path behavior;
|
- output path behavior;
|
||||||
@@ -127,7 +127,7 @@ Once implemented, `docs/cli.md` should document:
|
|||||||
|
|
||||||
- input module selection and module-specific options;
|
- input module selection and module-specific options;
|
||||||
- chunk module selection and module-specific options;
|
- chunk module selection and module-specific options;
|
||||||
- process module selection and module-specific options;
|
- extract module selection and module-specific options;
|
||||||
- merge module selection and module-specific options;
|
- merge module selection and module-specific options;
|
||||||
- normalize module selection and module-specific options;
|
- normalize module selection and module-specific options;
|
||||||
- output module selection and module-specific options;
|
- output module selection and module-specific options;
|
||||||
@@ -176,7 +176,7 @@ Before merging docs, check:
|
|||||||
|
|
||||||
- Does the document describe implemented behavior outside `docs/roadmap/`?
|
- Does the document describe implemented behavior outside `docs/roadmap/`?
|
||||||
- Are source-format details isolated to input module or integration docs?
|
- Are source-format details isolated to input module or integration docs?
|
||||||
- Are D&D details isolated to process module or artifact docs?
|
- Are D&D details isolated to extract module or artifact docs?
|
||||||
- Is there one canonical home for the topic?
|
- Is there one canonical home for the topic?
|
||||||
- Do command examples match implemented CLI syntax?
|
- Do command examples match implemented CLI syntax?
|
||||||
- Are examples valid, maintained, and free of secrets?
|
- Are examples valid, maintained, and free of secrets?
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ This is a staged implementation plan for
|
|||||||
an LLM coding agent to follow stage by stage.
|
an LLM coding agent to follow stage by stage.
|
||||||
|
|
||||||
This plan implements only checkpoint 2. Do not implement real input modules,
|
This plan implements only checkpoint 2. Do not implement real input modules,
|
||||||
real process modules, source chunking, LLM provider clients, prompt assets,
|
real extract modules, source chunking, LLM provider clients, prompt assets,
|
||||||
response schema registries, diagnostics run directories, production config
|
response schema registries, diagnostics run directories, production config
|
||||||
loading, or D&D artifact schemas in this pass.
|
loading, or D&D artifact schemas in this pass.
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ Required boundaries:
|
|||||||
|
|
||||||
- framework packages must stay source-agnostic and domain-agnostic;
|
- framework packages must stay source-agnostic and domain-agnostic;
|
||||||
- input adapter registry code must not import concrete input module packages;
|
- input adapter registry code must not import concrete input module packages;
|
||||||
- extractor registry and runner code must not import concrete process module
|
- extractor registry and runner code must not import concrete extract module
|
||||||
packages;
|
packages;
|
||||||
- runner code should operate on `SourceDocument`, not transcript-specific
|
- runner code should operate on `SourceDocument`, not transcript-specific
|
||||||
structures;
|
structures;
|
||||||
@@ -64,26 +64,26 @@ Add a constructor-based registry for `contracts.InputAdapter`.
|
|||||||
|
|
||||||
### Files To Add
|
### Files To Add
|
||||||
|
|
||||||
- `internal/framework/inputregistry/registry.go`
|
- `internal/framework/pipeline/input_registry.go`
|
||||||
- `internal/framework/inputregistry/registry_test.go`
|
- `internal/framework/pipeline/input_registry_test.go`
|
||||||
|
|
||||||
### Required API
|
### Required API
|
||||||
|
|
||||||
Create package `inputregistry`.
|
Extend package `pipeline`.
|
||||||
|
|
||||||
Define:
|
Define:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
type Constructor func() (contracts.InputAdapter, error)
|
type InputAdapterConstructor func() (contracts.InputAdapter, error)
|
||||||
|
|
||||||
type Registry struct {
|
type InputAdapterRegistry struct {
|
||||||
// unexported fields
|
// unexported fields
|
||||||
}
|
}
|
||||||
|
|
||||||
func New() *Registry
|
func NewInputAdapterRegistry() *InputAdapterRegistry
|
||||||
func (r *Registry) Register(key string, constructor Constructor) error
|
func (r *InputAdapterRegistry) Register(key string, constructor InputAdapterConstructor) error
|
||||||
func (r *Registry) Build(key string) (contracts.InputAdapter, error)
|
func (r *InputAdapterRegistry) Build(key string) (contracts.InputAdapter, error)
|
||||||
func (r *Registry) RegisteredKeys() []string
|
func (r *InputAdapterRegistry) RegisteredKeys() []string
|
||||||
```
|
```
|
||||||
|
|
||||||
### Required Behavior
|
### Required Behavior
|
||||||
@@ -137,8 +137,8 @@ Use fake adapters only. Do not add concrete input module packages.
|
|||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/inputregistry
|
gofmt -w internal/framework/pipeline
|
||||||
go test ./internal/framework/inputregistry
|
go test ./internal/framework/pipeline
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -150,31 +150,31 @@ Add a constructor-based registry for `contracts.Extractor`.
|
|||||||
|
|
||||||
### Files To Add
|
### Files To Add
|
||||||
|
|
||||||
- `internal/framework/extractorregistry/registry.go`
|
- `internal/framework/pipeline/extractor_registry.go`
|
||||||
- `internal/framework/extractorregistry/registry_test.go`
|
- `internal/framework/pipeline/extractor_registry_test.go`
|
||||||
|
|
||||||
### Required API
|
### Required API
|
||||||
|
|
||||||
Create package `extractorregistry`.
|
Extend package `pipeline`.
|
||||||
|
|
||||||
Define:
|
Define:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
type Constructor func() (contracts.Extractor, error)
|
type ExtractorConstructor func() (contracts.Extractor, error)
|
||||||
|
|
||||||
type Registry struct {
|
type ExtractorRegistry struct {
|
||||||
// unexported fields
|
// unexported fields
|
||||||
}
|
}
|
||||||
|
|
||||||
func New() *Registry
|
func NewExtractorRegistry() *ExtractorRegistry
|
||||||
func (r *Registry) Register(key string, constructor Constructor) error
|
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error
|
||||||
func (r *Registry) Build(key string) (contracts.Extractor, error)
|
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error)
|
||||||
func (r *Registry) RegisteredKeys() []string
|
func (r *ExtractorRegistry) RegisteredKeys() []string
|
||||||
```
|
```
|
||||||
|
|
||||||
### Required Behavior
|
### Required Behavior
|
||||||
|
|
||||||
Mirror `inputregistry` behavior, but for `contracts.Extractor`.
|
Mirror input adapter registry behavior, but for `contracts.Extractor`.
|
||||||
|
|
||||||
`Build` must reject extractors whose `Key()` does not match the normalized key.
|
`Build` must reject extractors whose `Key()` does not match the normalized key.
|
||||||
It should not validate artifact type, schema version, or validator chain. Those
|
It should not validate artifact type, schema version, or validator chain. Those
|
||||||
@@ -196,15 +196,15 @@ Cover the same cases as the input adapter registry:
|
|||||||
- sorted `RegisteredKeys`;
|
- sorted `RegisteredKeys`;
|
||||||
- nil registry behavior.
|
- nil registry behavior.
|
||||||
|
|
||||||
Use fake extractors only. Do not add concrete process module packages.
|
Use fake extractors only. Do not add concrete extract module packages.
|
||||||
|
|
||||||
### Validation
|
### Validation
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/extractorregistry
|
gofmt -w internal/framework/pipeline
|
||||||
go test ./internal/framework/extractorregistry
|
go test ./internal/framework/pipeline
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -216,12 +216,12 @@ Add shared validator decision helpers and decision-cardinality enforcement.
|
|||||||
|
|
||||||
### Files To Add
|
### Files To Add
|
||||||
|
|
||||||
- `internal/framework/validators/validators.go`
|
- `internal/framework/validate/validate.go`
|
||||||
- `internal/framework/validators/validators_test.go`
|
- `internal/framework/validate/validate_test.go`
|
||||||
|
|
||||||
### Required API
|
### Required API
|
||||||
|
|
||||||
Create package `validators`.
|
Create package `validate`.
|
||||||
|
|
||||||
Define reason constants:
|
Define reason constants:
|
||||||
|
|
||||||
@@ -236,7 +236,7 @@ Define helpers:
|
|||||||
```go
|
```go
|
||||||
func Approved(candidateIndex int) contracts.ValidationDecision
|
func Approved(candidateIndex int) contracts.ValidationDecision
|
||||||
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision
|
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision
|
||||||
func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []contracts.ValidationDecision) error
|
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error
|
||||||
```
|
```
|
||||||
|
|
||||||
### Required Behavior
|
### Required Behavior
|
||||||
@@ -264,8 +264,8 @@ func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []co
|
|||||||
- work with candidate indices, not slice positions;
|
- work with candidate indices, not slice positions;
|
||||||
- allow an empty candidate slice only when decisions are also empty.
|
- allow an empty candidate slice only when decisions are also empty.
|
||||||
|
|
||||||
Candidate indices are expected to be unique by the time validators run. If the
|
Artifact candidate indices are expected to be unique by the time validators run.
|
||||||
candidate slice itself contains duplicate indices, return an error.
|
If the candidate slice itself contains duplicate indices, return an error.
|
||||||
|
|
||||||
### Required Tests
|
### Required Tests
|
||||||
|
|
||||||
@@ -285,8 +285,8 @@ Cover:
|
|||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/validators
|
gofmt -w internal/framework/validate
|
||||||
go test ./internal/framework/validators
|
go test ./internal/framework/validate
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -294,17 +294,17 @@ go test ./...
|
|||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
Add runner package types and constructor without implementing the full run loop
|
Add pipeline runner types and constructor without implementing the full run loop
|
||||||
yet.
|
yet.
|
||||||
|
|
||||||
### Files To Add
|
### Files To Add
|
||||||
|
|
||||||
- `internal/framework/runner/runner.go`
|
- `internal/framework/pipeline/runner.go`
|
||||||
- `internal/framework/runner/runner_test.go`
|
- `internal/framework/pipeline/runner_test.go`
|
||||||
|
|
||||||
### Required API
|
### Required API
|
||||||
|
|
||||||
Create package `runner`.
|
Create or extend package `pipeline`.
|
||||||
|
|
||||||
Define:
|
Define:
|
||||||
|
|
||||||
@@ -364,8 +364,8 @@ assertions. Runner should depend only on framework contracts and core packages.
|
|||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/runner
|
gofmt -w internal/framework/pipeline
|
||||||
go test ./internal/framework/runner
|
go test ./internal/framework/pipeline
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -378,8 +378,8 @@ artifacts.
|
|||||||
|
|
||||||
### Files To Update
|
### Files To Update
|
||||||
|
|
||||||
- `internal/framework/runner/runner.go`
|
- `internal/framework/pipeline/runner.go`
|
||||||
- `internal/framework/runner/runner_test.go`
|
- `internal/framework/pipeline/runner_test.go`
|
||||||
|
|
||||||
### Required API
|
### Required API
|
||||||
|
|
||||||
@@ -396,7 +396,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error)
|
|||||||
1. reject nil runner or nil extractor factory;
|
1. reject nil runner or nil extractor factory;
|
||||||
2. validate `input.Source` with `source.ValidateDocument`;
|
2. validate `input.Source` with `source.ValidateDocument`;
|
||||||
3. reject an empty `ExtractorKeys` list;
|
3. reject an empty `ExtractorKeys` list;
|
||||||
4. process extractor keys in input order;
|
4. iterate extractor keys in input order;
|
||||||
5. build each extractor through the factory;
|
5. build each extractor through the factory;
|
||||||
6. call `Extractor.Extract` with the source, LLM client, and metadata;
|
6. call `Extractor.Extract` with the source, LLM client, and metadata;
|
||||||
7. append extractor warnings to `RunOutput.Warnings`;
|
7. append extractor warnings to `RunOutput.Warnings`;
|
||||||
@@ -406,9 +406,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error)
|
|||||||
11. reject candidates denied by validators;
|
11. reject candidates denied by validators;
|
||||||
12. return approved/rejected artifacts in deterministic order.
|
12. return approved/rejected artifacts in deterministic order.
|
||||||
|
|
||||||
Candidate normalization must:
|
Artifact candidate normalization must:
|
||||||
|
|
||||||
- assign `Candidate.Index` using a global monotonically increasing counter;
|
- assign `ArtifactCandidate.Index` using a global monotonically increasing counter;
|
||||||
- fill empty `ExtractorKey`, `ArtifactType`, and `SchemaVersion` from the
|
- fill empty `ExtractorKey`, `ArtifactType`, and `SchemaVersion` from the
|
||||||
owning extractor;
|
owning extractor;
|
||||||
- return an error if a candidate's non-empty `ExtractorKey` differs from
|
- return an error if a candidate's non-empty `ExtractorKey` differs from
|
||||||
@@ -423,7 +423,7 @@ Validation behavior:
|
|||||||
- if an extractor has no validators, all normalized candidates are approved;
|
- if an extractor has no validators, all normalized candidates are approved;
|
||||||
- each validator receives only currently eligible candidates;
|
- each validator receives only currently eligible candidates;
|
||||||
- each validator result must have `ValidatorName == Validator.Name()`;
|
- each validator result must have `ValidatorName == Validator.Name()`;
|
||||||
- call `validators.EnforceDecisionCardinality` for every validator result;
|
- call `validate.EnforceDecisionCardinality` for every validator result;
|
||||||
- approved decisions keep candidates eligible for the next validator;
|
- approved decisions keep candidates eligible for the next validator;
|
||||||
- rejected decisions create `artifacts.RejectedArtifact` records using the
|
- rejected decisions create `artifacts.RejectedArtifact` records using the
|
||||||
candidate, validator name, reason code, and message;
|
candidate, validator name, reason code, and message;
|
||||||
@@ -469,8 +469,8 @@ Use fake extractors, validators, and factories only.
|
|||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/runner
|
gofmt -w internal/framework/pipeline
|
||||||
go test ./internal/framework/runner
|
go test ./internal/framework/pipeline
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -478,16 +478,16 @@ go test ./...
|
|||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
Prove the extractor registry and runner compose without adding real process
|
Prove the extractor registry and runner compose without adding real extract
|
||||||
modules.
|
modules.
|
||||||
|
|
||||||
### Files To Add
|
### Files To Add
|
||||||
|
|
||||||
- `internal/framework/runner/registry_integration_test.go`
|
- `internal/framework/pipeline/registry_integration_test.go`
|
||||||
|
|
||||||
### Required Test Scenario
|
### Required Test Scenario
|
||||||
|
|
||||||
Use `extractorregistry.New()` to register two fake extractor constructors.
|
Use `NewExtractorRegistry()` to register two fake extractor constructors.
|
||||||
|
|
||||||
Run the runner with:
|
Run the runner with:
|
||||||
|
|
||||||
@@ -502,15 +502,15 @@ Assertions:
|
|||||||
- extractor execution follows configured key order;
|
- extractor execution follows configured key order;
|
||||||
- approved artifacts are in deterministic order;
|
- approved artifacts are in deterministic order;
|
||||||
- rejected artifacts are in deterministic order;
|
- rejected artifacts are in deterministic order;
|
||||||
- no concrete input module or process module packages are imported.
|
- no concrete input module or extract module packages are imported.
|
||||||
|
|
||||||
### Validation
|
### Validation
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -w internal/framework/runner
|
gofmt -w internal/framework/pipeline
|
||||||
go test ./internal/framework/runner
|
go test ./internal/framework/pipeline
|
||||||
go test ./...
|
go test ./...
|
||||||
go build ./cmd/notarius
|
go build ./cmd/notarius
|
||||||
```
|
```
|
||||||
@@ -528,7 +528,7 @@ Check:
|
|||||||
|
|
||||||
- no package names mention transcripts, Seriatim, D&D, spells, NPCs, items, or
|
- no package names mention transcripts, Seriatim, D&D, spells, NPCs, items, or
|
||||||
combat;
|
combat;
|
||||||
- no concrete input module or process module package exists;
|
- no concrete input module or extract module package exists;
|
||||||
- no LLM provider code exists;
|
- no LLM provider code exists;
|
||||||
- no prompt, response schema, diagnostics, config, or source chunking package
|
- no prompt, response schema, diagnostics, config, or source chunking package
|
||||||
was added;
|
was added;
|
||||||
|
|||||||
@@ -14,18 +14,18 @@ The first MVP should target audio transcripts generated by Seriatim. That
|
|||||||
choice should be implemented as an input-stage module, not as a
|
choice should be implemented as an input-stage module, not as a
|
||||||
transcript-specific assumption in the application core. Later input sources,
|
transcript-specific assumption in the application core. Later input sources,
|
||||||
such as unstructured Markdown notes or Obsidian documents, should be addable
|
such as unstructured Markdown notes or Obsidian documents, should be addable
|
||||||
through new input and process modules without reshaping the framework.
|
through new input and extract modules without reshaping the framework.
|
||||||
|
|
||||||
The first extraction domain should be D&D session analysis, starting with spell
|
The first extraction domain should be D&D session analysis, starting with spell
|
||||||
casts. That domain should live in process-stage modules and related schemas, not
|
casts. That domain should live in extract-stage modules and related schemas, not
|
||||||
in core framework packages.
|
in core framework packages.
|
||||||
|
|
||||||
The application should follow the same broad architecture as Audita:
|
The application should follow the same broad architecture as Audita:
|
||||||
|
|
||||||
- deterministic core packages for config, source documents, artifacts, diagnostics, and reporting;
|
- deterministic core packages for source documents, artifacts, and configuration once needed;
|
||||||
- input-stage modules that translate external source formats into a small internal source model;
|
- input-stage modules that translate external source formats into a small internal source model;
|
||||||
- reusable framework packages for contracts, orchestration, LLM runtime, structured output, and validation;
|
- reusable framework packages for contracts, orchestration, LLM runtime, structured output, and validation;
|
||||||
- independent process-stage modules that own domain-specific behavior;
|
- independent extract-stage modules that own domain-specific behavior;
|
||||||
- independent validator packages;
|
- independent validator packages;
|
||||||
- embedded prompt and JSON schema assets;
|
- embedded prompt and JSON schema assets;
|
||||||
- CLI orchestration that wires the pieces together without owning domain logic.
|
- CLI orchestration that wires the pieces together without owning domain logic.
|
||||||
@@ -37,7 +37,7 @@ artifacts rather than proposing and applying transcript corrections.
|
|||||||
|
|
||||||
- Keep the core input model generic: ordered text units plus metadata.
|
- Keep the core input model generic: ordered text units plus metadata.
|
||||||
- Keep source-format details in hexagonal input modules.
|
- Keep source-format details in hexagonal input modules.
|
||||||
- Keep extraction-domain details in process modules.
|
- Keep extraction-domain details in extract modules.
|
||||||
- Treat evidence as source references, not transcript references.
|
- Treat evidence as source references, not transcript references.
|
||||||
- Prefer narrow, useful abstractions over a universal document model.
|
- Prefer narrow, useful abstractions over a universal document model.
|
||||||
- Preserve enough provenance for validation, replay, and downstream inspection.
|
- Preserve enough provenance for validation, replay, and downstream inspection.
|
||||||
@@ -48,28 +48,14 @@ artifacts rather than proposing and applying transcript corrections.
|
|||||||
cmd/notarius
|
cmd/notarius
|
||||||
internal/cli
|
internal/cli
|
||||||
|
|
||||||
internal/core/config
|
|
||||||
internal/core/source
|
internal/core/source
|
||||||
internal/core/sourcechunking
|
|
||||||
internal/core/artifacts
|
internal/core/artifacts
|
||||||
internal/core/diagnostics
|
|
||||||
internal/core/reporting
|
|
||||||
internal/core/extractorcatalog
|
|
||||||
internal/core/inputcatalog
|
|
||||||
|
|
||||||
internal/framework/contracts
|
internal/framework/contracts
|
||||||
internal/framework/extraction
|
|
||||||
internal/framework/runner
|
|
||||||
internal/framework/pipeline
|
internal/framework/pipeline
|
||||||
internal/framework/merge
|
internal/framework/validate
|
||||||
internal/framework/normalize
|
|
||||||
internal/framework/output
|
|
||||||
internal/framework/validators
|
|
||||||
internal/framework/llm
|
internal/framework/llm
|
||||||
internal/framework/responseschema
|
internal/framework/prompt
|
||||||
internal/framework/structuredoutput
|
|
||||||
internal/framework/promptcontext
|
|
||||||
internal/framework/warnings
|
|
||||||
|
|
||||||
internal/modules/input/seriatim
|
internal/modules/input/seriatim
|
||||||
internal/modules/input/markdown
|
internal/modules/input/markdown
|
||||||
@@ -77,10 +63,10 @@ internal/modules/input/markdown
|
|||||||
internal/modules/chunk/generic
|
internal/modules/chunk/generic
|
||||||
internal/modules/chunk/dndtranscript
|
internal/modules/chunk/dndtranscript
|
||||||
|
|
||||||
internal/modules/process/dnd/spells
|
internal/modules/extract/dnd/spells
|
||||||
internal/modules/process/dnd/items
|
internal/modules/extract/dnd/items
|
||||||
internal/modules/process/dnd/npcs
|
internal/modules/extract/dnd/npcs
|
||||||
internal/modules/process/dnd/combat
|
internal/modules/extract/dnd/combat
|
||||||
|
|
||||||
internal/modules/merge/appendorder
|
internal/modules/merge/appendorder
|
||||||
internal/modules/merge/dnd/spells
|
internal/modules/merge/dnd/spells
|
||||||
@@ -95,7 +81,6 @@ internal/validators/schema_validity
|
|||||||
internal/validators/domain_consistency
|
internal/validators/domain_consistency
|
||||||
internal/validators/llm_review
|
internal/validators/llm_review
|
||||||
|
|
||||||
internal/prompts
|
|
||||||
examples
|
examples
|
||||||
docs/internal
|
docs/internal
|
||||||
```
|
```
|
||||||
@@ -104,6 +89,15 @@ The `markdown` input module and D&D-specific chunk, merge, normalize, and
|
|||||||
output modules are listed as likely future packages. The MVP should implement
|
output modules are listed as likely future packages. The MVP should implement
|
||||||
only the stage modules needed by the checkpoint sequence.
|
only the stage modules needed by the checkpoint sequence.
|
||||||
|
|
||||||
|
`internal/core/config` should be added when production configuration exists.
|
||||||
|
|
||||||
|
The framework package list is intentionally consolidated. `pipeline` should own
|
||||||
|
runner orchestration, stage registries, and small merge/normalize/output helpers
|
||||||
|
until those boundaries prove they need separate packages. `llm` should own
|
||||||
|
structured output and response-schema mechanics until those concerns become too
|
||||||
|
large or import-heavy. `prompt` should own prompt assets and rendering helpers
|
||||||
|
once prompt assets exist.
|
||||||
|
|
||||||
## Core Concepts
|
## Core Concepts
|
||||||
|
|
||||||
### SourceDocument
|
### SourceDocument
|
||||||
@@ -138,6 +132,13 @@ Initial source-unit assumptions:
|
|||||||
- adapter-specific metadata may carry speaker, timestamps, heading paths, page
|
- adapter-specific metadata may carry speaker, timestamps, heading paths, page
|
||||||
numbers, or other source details.
|
numbers, or other source details.
|
||||||
|
|
||||||
|
Core source metadata should remain `map[string]any`. Notarius should not define
|
||||||
|
a universal document model. Instead, the project should document well-known
|
||||||
|
metadata keys, such as `speaker`, `start`, `end`, and `heading_path`, as
|
||||||
|
conventions. Input modules may export typed accessor helpers for their own
|
||||||
|
metadata, such as `seriatim.SpeakerOf(unit)`, without leaking those helpers into
|
||||||
|
core framework contracts.
|
||||||
|
|
||||||
### Input Module / Adapter Contract
|
### Input Module / Adapter Contract
|
||||||
|
|
||||||
Hexagonal boundary for external source formats.
|
Hexagonal boundary for external source formats.
|
||||||
@@ -183,6 +184,11 @@ Initial source-reference validation should require:
|
|||||||
Transcript-oriented output can still present these as transcript segment ranges
|
Transcript-oriented output can still present these as transcript segment ranges
|
||||||
when the adapter metadata makes that interpretation available.
|
when the adapter metadata makes that interpretation available.
|
||||||
|
|
||||||
|
Source references should preserve the exact ranges produced by extractors and
|
||||||
|
validators. Overlapping ranges should not be merged or rewritten by generic
|
||||||
|
pipeline code. If a domain module wants a derived compact range later, that
|
||||||
|
should be additional output, not a replacement for the original evidence.
|
||||||
|
|
||||||
### Extractor
|
### Extractor
|
||||||
|
|
||||||
Reusable module contract for producing one artifact type.
|
Reusable module contract for producing one artifact type.
|
||||||
@@ -193,7 +199,7 @@ type Extractor interface {
|
|||||||
ArtifactType() string
|
ArtifactType() string
|
||||||
SchemaVersion() string
|
SchemaVersion() string
|
||||||
Validators() []Validator
|
Validators() []Validator
|
||||||
Process(ctx context.Context, req ProcessRequest) (ProcessResult, error)
|
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -201,9 +207,16 @@ An extractor should receive either a whole source document or a source chunk,
|
|||||||
depending on processing mode. It should return typed artifact candidates plus
|
depending on processing mode. It should return typed artifact candidates plus
|
||||||
warnings. It should not mutate the source document.
|
warnings. It should not mutate the source document.
|
||||||
|
|
||||||
Process modules own domain concepts. For example, D&D spell extraction should
|
`ExtractionRequest` should be designed now to carry both the active chunk and
|
||||||
live under `internal/modules/process/dnd/spells`; a future to-do extractor for
|
optional ambient context, even if the MVP leaves that context empty. Useful
|
||||||
notes should live under a different process-module path and use the same
|
ambient context may include a document synopsis, prior-chunk summaries, known
|
||||||
|
entities, or other module-provided state. D&D spell extraction can likely work
|
||||||
|
per chunk, but combat, NPC, and identity-oriented extraction will need broader
|
||||||
|
context. Adding the field later would force churn across every extractor.
|
||||||
|
|
||||||
|
Extract modules own domain concepts. For example, D&D spell extraction should
|
||||||
|
live under `internal/modules/extract/dnd/spells`; a future to-do extractor for
|
||||||
|
notes should live under a different extract-module path and use the same
|
||||||
framework contract.
|
framework contract.
|
||||||
|
|
||||||
### Chunker
|
### Chunker
|
||||||
@@ -247,14 +260,41 @@ Validators should cover:
|
|||||||
Validator output should follow Audita's decision-cardinality model: each
|
Validator output should follow Audita's decision-cardinality model: each
|
||||||
candidate artifact receives exactly one decision per validator.
|
candidate artifact receives exactly one decision per validator.
|
||||||
|
|
||||||
|
LLM-backed review should be modeled as part of a module's validator chain, not
|
||||||
|
as a separate global review phase. Extract modules should be able to attach one
|
||||||
|
or more deterministic or LLM-backed validators. Normalize-stage modules may also
|
||||||
|
run validator chains, including LLM-backed validators, when semantic
|
||||||
|
reconciliation needs review.
|
||||||
|
|
||||||
### Artifact
|
### Artifact
|
||||||
|
|
||||||
Final approved JSON output from one or more extractors.
|
Final approved JSON output from one or more extractors.
|
||||||
|
|
||||||
Artifacts should preserve enough metadata to support downstream validation,
|
Artifacts should preserve enough metadata to support downstream validation,
|
||||||
debugging, and replay. The exact top-level envelope is still open, but should
|
debugging, and replay.
|
||||||
include artifact type, schema version, extracted records, source references, and
|
|
||||||
run manifest data.
|
The pipeline should carry artifact candidates through a generic envelope with a
|
||||||
|
`json.RawMessage` payload. Extract modules should own typed Go structs at their
|
||||||
|
module boundary, then encode those typed records into the generic artifact
|
||||||
|
candidate envelope before returning to framework code. This keeps stage
|
||||||
|
contracts simple and avoids generic type plumbing across unrelated artifact
|
||||||
|
families.
|
||||||
|
|
||||||
|
Final durable output should be one file per artifact type plus a run-level
|
||||||
|
manifest/index file. This supports partial success and lets downstream consumers
|
||||||
|
read only the artifact types they need. Each artifact file should include its
|
||||||
|
artifact type, extractor key, extractor schema version, envelope format version,
|
||||||
|
records, source references, and enough provenance to connect it to the run
|
||||||
|
manifest.
|
||||||
|
|
||||||
|
Every artifact record should require source references unless that artifact
|
||||||
|
schema explicitly opts into ungrounded fields. Artifact-level metadata, counts,
|
||||||
|
run information, and other derived summary fields are exempt from the per-record
|
||||||
|
grounding rule.
|
||||||
|
|
||||||
|
Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||||
|
format version. A single global schema version would couple unrelated extractor
|
||||||
|
release cadence.
|
||||||
|
|
||||||
### RunManifest
|
### RunManifest
|
||||||
|
|
||||||
@@ -262,9 +302,14 @@ Per-run provenance record.
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
InputAdapter string `json:"input_adapter"`
|
EnvelopeVersion string `json:"envelope_version"`
|
||||||
|
InputModule string `json:"input_module"`
|
||||||
|
Chunker string `json:"chunker"`
|
||||||
SourceDigests []string `json:"source_digests"`
|
SourceDigests []string `json:"source_digests"`
|
||||||
Extractors []string `json:"extractors"`
|
Extractors []string `json:"extractors"`
|
||||||
|
Merger string `json:"merger"`
|
||||||
|
Normalizer string `json:"normalizer"`
|
||||||
|
OutputEncoder string `json:"output_encoder"`
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
ValidationStatus string `json:"validation_status"`
|
ValidationStatus string `json:"validation_status"`
|
||||||
}
|
}
|
||||||
@@ -345,7 +390,7 @@ The architecture should support extractors outside the D&D domain. Examples:
|
|||||||
- decisions and action items from meeting transcripts;
|
- decisions and action items from meeting transcripts;
|
||||||
- named people, places, and dates from research notes.
|
- named people, places, and dates from research notes.
|
||||||
|
|
||||||
These should be addable as process modules without changing runner,
|
These should be addable as extract modules without changing runner,
|
||||||
validator, source-reference, or LLM framework contracts.
|
validator, source-reference, or LLM framework contracts.
|
||||||
|
|
||||||
## Proposed Pipeline Flow
|
## Proposed Pipeline Flow
|
||||||
@@ -353,7 +398,7 @@ validator, source-reference, or LLM framework contracts.
|
|||||||
The application workflow should be first-class:
|
The application workflow should be first-class:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
Proposed runner flow:
|
Proposed runner flow:
|
||||||
@@ -367,7 +412,7 @@ Proposed runner flow:
|
|||||||
7. Resolve the configured chunker.
|
7. Resolve the configured chunker.
|
||||||
8. Chunk source units into deterministic source chunks.
|
8. Chunk source units into deterministic source chunks.
|
||||||
9. Resolve configured extractor instances through a registry.
|
9. Resolve configured extractor instances through a registry.
|
||||||
10. Process chunks in extractor-defined mode.
|
10. Extract from chunks in extractor-defined mode.
|
||||||
11. Merge per-chunk artifact candidates deterministically.
|
11. Merge per-chunk artifact candidates deterministically.
|
||||||
12. Normalize merged artifact candidates.
|
12. Normalize merged artifact candidates.
|
||||||
13. Run deterministic validators before LLM-backed validators.
|
13. Run deterministic validators before LLM-backed validators.
|
||||||
@@ -397,6 +442,11 @@ Reuse these architectural patterns:
|
|||||||
- validator decision cardinality and deterministic validator ordering;
|
- validator decision cardinality and deterministic validator ordering;
|
||||||
- CLI tests and fixture-driven integration tests.
|
- CLI tests and fixture-driven integration tests.
|
||||||
|
|
||||||
|
The fixture-driven integration-test pattern should begin at checkpoint 3 with a
|
||||||
|
walking skeleton over fake modules and a fake LLM client. Later checkpoints
|
||||||
|
should replace fake pieces with real Seriatim, runtime, and D&D modules without
|
||||||
|
losing that end-to-end contract coverage.
|
||||||
|
|
||||||
Avoid copying these Audita concepts directly:
|
Avoid copying these Audita concepts directly:
|
||||||
|
|
||||||
- transcript-specific core types;
|
- transcript-specific core types;
|
||||||
@@ -422,32 +472,47 @@ compiling and targeted tests covering the newly introduced contracts or behavior
|
|||||||
5. [Seriatim Input Module](5-seriatim-input-module.md)
|
5. [Seriatim Input Module](5-seriatim-input-module.md)
|
||||||
6. [D&D Spells Extractor](6-dnd-spells-extractor.md)
|
6. [D&D Spells Extractor](6-dnd-spells-extractor.md)
|
||||||
|
|
||||||
The first useful vertical slice should arrive at checkpoint 6: Seriatim
|
The first contract-level walking skeleton should arrive at checkpoint 3: fixture
|
||||||
transcript input to validated D&D spell artifact output. Earlier checkpoints are
|
input through fake input, chunk, extract, merge, normalize, and output modules
|
||||||
intentionally contract-first and may not produce useful user output yet.
|
with a fake LLM client. The first useful vertical slice should arrive at
|
||||||
|
checkpoint 6: Seriatim transcript input to validated D&D spell artifact output.
|
||||||
|
Earlier checkpoints remain contract-first and may not produce useful user output
|
||||||
|
yet.
|
||||||
|
|
||||||
|
## Architecture Decisions
|
||||||
|
|
||||||
|
- Final durable output should use one artifact file per artifact type plus a
|
||||||
|
run-level manifest/index file.
|
||||||
|
- Framework artifact flow should use a generic envelope with `json.RawMessage`
|
||||||
|
payloads. Extract modules should use typed Go structs at their own boundaries.
|
||||||
|
- Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||||
|
format version.
|
||||||
|
- Artifact records should require source references by default. Individual
|
||||||
|
schemas may explicitly opt into ungrounded fields. Artifact-level metadata is
|
||||||
|
exempt.
|
||||||
|
- Source-reference ranges should be preserved exactly. Generic pipeline code
|
||||||
|
should not merge or rewrite overlapping ranges.
|
||||||
|
- `ExtractionRequest` should carry the active chunk plus optional ambient
|
||||||
|
context for document synopsis, prior-chunk summaries, known entities, or
|
||||||
|
similar module-provided state.
|
||||||
|
- LLM-backed review should be part of module-owned validator chains. Extract
|
||||||
|
modules and normalize modules may both use deterministic and LLM-backed
|
||||||
|
validators.
|
||||||
|
- The Seriatim MVP should support only the minimal Seriatim schema. Broader
|
||||||
|
Seriatim schema support should be added later without changing core source
|
||||||
|
contracts.
|
||||||
|
- Core source metadata should remain `map[string]any`. Well-known metadata keys
|
||||||
|
should be documented as conventions, and input modules may expose typed
|
||||||
|
accessor helpers for their own metadata.
|
||||||
|
|
||||||
## Open Design Questions
|
## Open Design Questions
|
||||||
|
|
||||||
- Should final output be one combined artifact envelope or one file per
|
- Which artifact types should use generic append-in-chunk-order merge, and which
|
||||||
extractor?
|
should use domain-specific merge?
|
||||||
- Should extractor output use typed Go structs per artifact or a generic
|
|
||||||
artifact record with `json.RawMessage` payloads?
|
|
||||||
- Should schemas be versioned per extractor, globally, or both?
|
|
||||||
- Should every record require source references, or should some top-level
|
|
||||||
artifact metadata be allowed without source references?
|
|
||||||
- Should overlapping source-reference ranges be merged, preserved exactly, or
|
|
||||||
both?
|
|
||||||
- Should extraction run independently per source chunk only, or should some
|
|
||||||
extractors receive whole-document context?
|
|
||||||
- Which artifact types can use a generic append-in-chunk-order merger?
|
|
||||||
- Which artifact types need domain-specific normalization for deduplication,
|
- Which artifact types need domain-specific normalization for deduplication,
|
||||||
identity resolution, or consistency?
|
identity resolution, or consistency?
|
||||||
- Should LLM review be part of each extractor's validator chain or a separate
|
- What should the configuration model look like for selecting input, chunk,
|
||||||
review phase?
|
extract, merge, normalize, output, and validator modules?
|
||||||
- Should the Seriatim adapter accept only its minimal schema initially or also
|
|
||||||
support richer transcript schemas?
|
|
||||||
- Should source-unit metadata be untyped `map[string]any`, typed extension
|
|
||||||
structs, or both?
|
|
||||||
|
|
||||||
## Near-Term Documentation Tasks
|
## Near-Term Documentation Tasks
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Candidate struct {
|
type ArtifactCandidate struct {
|
||||||
Index int `json:"index"`
|
Index int `json:"index"`
|
||||||
ExtractorKey string `json:"extractor_key"`
|
ExtractorKey string `json:"extractor_key"`
|
||||||
ArtifactType string `json:"artifact_type"`
|
ArtifactType string `json:"artifact_type"`
|
||||||
@@ -27,24 +27,28 @@ type Artifact struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type RejectedArtifact struct {
|
type RejectedArtifact struct {
|
||||||
Candidate Candidate `json:"candidate"`
|
Candidate ArtifactCandidate `json:"candidate"`
|
||||||
ValidatorName string `json:"validator_name"`
|
ValidatorName string `json:"validator_name"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
InputAdapter string `json:"input_adapter,omitempty"`
|
InputModule string `json:"input_module,omitempty"`
|
||||||
|
Chunker string `json:"chunker,omitempty"`
|
||||||
SourceDigests []string `json:"source_digests,omitempty"`
|
SourceDigests []string `json:"source_digests,omitempty"`
|
||||||
Extractors []string `json:"extractors,omitempty"`
|
Extractors []string `json:"extractors,omitempty"`
|
||||||
|
Merger string `json:"merger,omitempty"`
|
||||||
|
Normalizer string `json:"normalizer,omitempty"`
|
||||||
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
SchemaVersion string `json:"schema_version,omitempty"`
|
SchemaVersion string `json:"schema_version,omitempty"`
|
||||||
ValidationStatus string `json:"validation_status,omitempty"`
|
ValidationStatus string `json:"validation_status,omitempty"`
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ArtifactFromCandidate(candidate Candidate) Artifact {
|
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
||||||
return Artifact{
|
return Artifact{
|
||||||
ExtractorKey: candidate.ExtractorKey,
|
ExtractorKey: candidate.ExtractorKey,
|
||||||
ArtifactType: candidate.ArtifactType,
|
ArtifactType: candidate.ArtifactType,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
||||||
candidate := Candidate{
|
candidate := ArtifactCandidate{
|
||||||
Index: 7,
|
Index: 7,
|
||||||
ExtractorKey: "generic-extractor",
|
ExtractorKey: "generic-extractor",
|
||||||
ArtifactType: "generic-artifact",
|
ArtifactType: "generic-artifact",
|
||||||
@@ -60,7 +60,7 @@ func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
||||||
candidate := Candidate{
|
candidate := ArtifactCandidate{
|
||||||
Index: 1,
|
Index: 1,
|
||||||
ExtractorKey: "generic-extractor",
|
ExtractorKey: "generic-extractor",
|
||||||
ArtifactType: "generic-artifact",
|
ArtifactType: "generic-artifact",
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ func (extractor compositionExtractor) Extract(ctx context.Context, req contracts
|
|||||||
}
|
}
|
||||||
|
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{
|
{
|
||||||
Index: 0,
|
Index: 0,
|
||||||
ExtractorKey: extractor.Key(),
|
ExtractorKey: extractor.Key(),
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ type ExtractionRequest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ExtractionResult struct {
|
type ExtractionResult struct {
|
||||||
Candidates []artifacts.Candidate `json:"candidates,omitempty"`
|
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Extractor interface {
|
type Extractor interface {
|
||||||
@@ -66,9 +66,9 @@ type Extractor interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ValidationRequest struct {
|
type ValidationRequest struct {
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
Candidates []artifacts.Candidate `json:"candidates"`
|
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ValidationDecision struct {
|
type ValidationDecision struct {
|
||||||
|
|||||||
@@ -58,19 +58,19 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
|
|||||||
|
|
||||||
candidate := result.Candidates[0]
|
candidate := result.Candidates[0]
|
||||||
if candidate.Index != 0 {
|
if candidate.Index != 0 {
|
||||||
t.Fatalf("Candidate.Index = %d, want 0", candidate.Index)
|
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
|
||||||
}
|
}
|
||||||
if candidate.ExtractorKey != extractor.Key() {
|
if candidate.ExtractorKey != extractor.Key() {
|
||||||
t.Fatalf("Candidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
|
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
|
||||||
}
|
}
|
||||||
if candidate.ArtifactType != extractor.ArtifactType() {
|
if candidate.ArtifactType != extractor.ArtifactType() {
|
||||||
t.Fatalf("Candidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
|
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
|
||||||
}
|
}
|
||||||
if candidate.SchemaVersion != extractor.SchemaVersion() {
|
if candidate.SchemaVersion != extractor.SchemaVersion() {
|
||||||
t.Fatalf("Candidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
|
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
|
||||||
}
|
}
|
||||||
if string(candidate.Payload) != `{"value":"example"}` {
|
if string(candidate.Payload) != `{"value":"example"}` {
|
||||||
t.Fatalf("Candidate.Payload = %s, want example payload", candidate.Payload)
|
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,7 +112,7 @@ func (extractor fakeExtractor) Validators() []Validator {
|
|||||||
|
|
||||||
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
|
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
|
||||||
return ExtractionResult{
|
return ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{
|
{
|
||||||
Index: 0,
|
Index: 0,
|
||||||
ExtractorKey: extractor.key,
|
ExtractorKey: extractor.key,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package extractorregistry
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -8,19 +8,19 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Constructor func() (contracts.Extractor, error)
|
type ExtractorConstructor func() (contracts.Extractor, error)
|
||||||
|
|
||||||
type Registry struct {
|
type ExtractorRegistry struct {
|
||||||
constructors map[string]Constructor
|
constructors map[string]ExtractorConstructor
|
||||||
}
|
}
|
||||||
|
|
||||||
func New() *Registry {
|
func NewExtractorRegistry() *ExtractorRegistry {
|
||||||
return &Registry{
|
return &ExtractorRegistry{
|
||||||
constructors: make(map[string]Constructor),
|
constructors: make(map[string]ExtractorConstructor),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) Register(key string, constructor Constructor) error {
|
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("extractor registry must not be nil")
|
return fmt.Errorf("extractor registry must not be nil")
|
||||||
}
|
}
|
||||||
@@ -40,7 +40,7 @@ func (r *Registry) Register(key string, constructor Constructor) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) Build(key string) (contracts.Extractor, error) {
|
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil, fmt.Errorf("extractor registry must not be nil")
|
return nil, fmt.Errorf("extractor registry must not be nil")
|
||||||
}
|
}
|
||||||
@@ -69,7 +69,7 @@ func (r *Registry) Build(key string) (contracts.Extractor, error) {
|
|||||||
return extractor, nil
|
return extractor, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) RegisteredKeys() []string {
|
func (r *ExtractorRegistry) RegisteredKeys() []string {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package extractorregistry
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -10,10 +10,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterAndBuild(t *testing.T) {
|
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err != nil {
|
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -26,10 +26,10 @@ func TestRegisterAndBuild(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
if err := registry.Register(" generic-extractor ", fakeConstructor("generic-extractor")); err != nil {
|
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,10 +42,10 @@ func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsEmptyKey(t *testing.T) {
|
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
err := registry.Register(" \t", fakeConstructor("generic-extractor"))
|
err := registry.Register(" \t", fakeExtractorConstructor("generic-extractor"))
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
@@ -55,13 +55,13 @@ func TestRegisterRejectsEmptyKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err != nil {
|
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := registry.Register(" generic-extractor ", fakeConstructor("generic-extractor"))
|
err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
@@ -71,8 +71,8 @@ func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsNilConstructor(t *testing.T) {
|
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
err := registry.Register("generic-extractor", nil)
|
err := registry.Register("generic-extractor", nil)
|
||||||
|
|
||||||
@@ -84,8 +84,8 @@ func TestRegisterRejectsNilConstructor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsUnknownKey(t *testing.T) {
|
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
_, err := registry.Build("missing-extractor")
|
_, err := registry.Build("missing-extractor")
|
||||||
|
|
||||||
@@ -97,8 +97,8 @@ func TestBuildRejectsUnknownKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildWrapsConstructorError(t *testing.T) {
|
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
constructorErr := errors.New("constructor failed")
|
constructorErr := errors.New("constructor failed")
|
||||||
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
||||||
return nil, constructorErr
|
return nil, constructorErr
|
||||||
@@ -119,8 +119,8 @@ func TestBuildWrapsConstructorError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsNilExtractor(t *testing.T) {
|
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -137,9 +137,9 @@ func TestBuildRejectsNilExtractor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
if err := registry.Register("generic-extractor", fakeConstructor("other-extractor")); err != nil {
|
if err := registry.Register("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,10 +153,10 @@ func TestBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
for _, key := range []string{"zeta", "alpha", "middle"} {
|
for _, key := range []string{"zeta", "alpha", "middle"} {
|
||||||
if err := registry.Register(key, fakeConstructor(key)); err != nil {
|
if err := registry.Register(key, fakeExtractorConstructor(key)); err != nil {
|
||||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -174,10 +174,10 @@ func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNilRegistryBehavior(t *testing.T) {
|
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
|
||||||
var registry *Registry
|
var registry *ExtractorRegistry
|
||||||
|
|
||||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err == nil {
|
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
}
|
}
|
||||||
if _, err := registry.Build("generic-extractor"); err == nil {
|
if _, err := registry.Build("generic-extractor"); err == nil {
|
||||||
@@ -188,8 +188,8 @@ func TestNilRegistryBehavior(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsEmptyKey(t *testing.T) {
|
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
_, err := registry.Build(" \n")
|
_, err := registry.Build(" \n")
|
||||||
|
|
||||||
@@ -201,32 +201,32 @@ func TestBuildRejectsEmptyKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeExtractor struct {
|
type registryFakeExtractor struct {
|
||||||
key string
|
key string
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeConstructor(key string) Constructor {
|
func fakeExtractorConstructor(key string) ExtractorConstructor {
|
||||||
return func() (contracts.Extractor, error) {
|
return func() (contracts.Extractor, error) {
|
||||||
return fakeExtractor{key: key}, nil
|
return registryFakeExtractor{key: key}, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) Key() string {
|
func (extractor registryFakeExtractor) Key() string {
|
||||||
return extractor.key
|
return extractor.key
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) ArtifactType() string {
|
func (extractor registryFakeExtractor) ArtifactType() string {
|
||||||
return "generic-artifact"
|
return "generic-artifact"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) SchemaVersion() string {
|
func (extractor registryFakeExtractor) SchemaVersion() string {
|
||||||
return "v1"
|
return "v1"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) Validators() []contracts.Validator {
|
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||||
return contracts.ExtractionResult{}, nil
|
return contracts.ExtractionResult{}, nil
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package inputregistry
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -8,19 +8,19 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Constructor func() (contracts.InputAdapter, error)
|
type InputAdapterConstructor func() (contracts.InputAdapter, error)
|
||||||
|
|
||||||
type Registry struct {
|
type InputAdapterRegistry struct {
|
||||||
constructors map[string]Constructor
|
constructors map[string]InputAdapterConstructor
|
||||||
}
|
}
|
||||||
|
|
||||||
func New() *Registry {
|
func NewInputAdapterRegistry() *InputAdapterRegistry {
|
||||||
return &Registry{
|
return &InputAdapterRegistry{
|
||||||
constructors: make(map[string]Constructor),
|
constructors: make(map[string]InputAdapterConstructor),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) Register(key string, constructor Constructor) error {
|
func (r *InputAdapterRegistry) Register(key string, constructor InputAdapterConstructor) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("input adapter registry must not be nil")
|
return fmt.Errorf("input adapter registry must not be nil")
|
||||||
}
|
}
|
||||||
@@ -40,7 +40,7 @@ func (r *Registry) Register(key string, constructor Constructor) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) Build(key string) (contracts.InputAdapter, error) {
|
func (r *InputAdapterRegistry) Build(key string) (contracts.InputAdapter, error) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil, fmt.Errorf("input adapter registry must not be nil")
|
return nil, fmt.Errorf("input adapter registry must not be nil")
|
||||||
}
|
}
|
||||||
@@ -69,7 +69,7 @@ func (r *Registry) Build(key string) (contracts.InputAdapter, error) {
|
|||||||
return adapter, nil
|
return adapter, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) RegisteredKeys() []string {
|
func (r *InputAdapterRegistry) RegisteredKeys() []string {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package inputregistry
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -11,10 +11,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterAndBuild(t *testing.T) {
|
func TestInputAdapterRegistryRegisterAndBuild(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err != nil {
|
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,10 +27,10 @@ func TestRegisterAndBuild(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
func TestInputAdapterRegistryRegisterAndBuildTrimKeys(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
if err := registry.Register(" generic-input ", fakeConstructor("generic-input")); err != nil {
|
if err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,10 +43,10 @@ func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsEmptyKey(t *testing.T) {
|
func TestInputAdapterRegistryRegisterRejectsEmptyKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
err := registry.Register(" \t", fakeConstructor("generic-input"))
|
err := registry.Register(" \t", fakeInputAdapterConstructor("generic-input"))
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
@@ -56,13 +56,13 @@ func TestRegisterRejectsEmptyKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
func TestInputAdapterRegistryRegisterRejectsDuplicateKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err != nil {
|
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := registry.Register(" generic-input ", fakeConstructor("generic-input"))
|
err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input"))
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
@@ -72,8 +72,8 @@ func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterRejectsNilConstructor(t *testing.T) {
|
func TestInputAdapterRegistryRegisterRejectsNilConstructor(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
err := registry.Register("generic-input", nil)
|
err := registry.Register("generic-input", nil)
|
||||||
|
|
||||||
@@ -85,8 +85,8 @@ func TestRegisterRejectsNilConstructor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsUnknownKey(t *testing.T) {
|
func TestInputAdapterRegistryBuildRejectsUnknownKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
_, err := registry.Build("missing-input")
|
_, err := registry.Build("missing-input")
|
||||||
|
|
||||||
@@ -98,8 +98,8 @@ func TestBuildRejectsUnknownKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildWrapsConstructorError(t *testing.T) {
|
func TestInputAdapterRegistryBuildWrapsConstructorError(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
constructorErr := errors.New("constructor failed")
|
constructorErr := errors.New("constructor failed")
|
||||||
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
||||||
return nil, constructorErr
|
return nil, constructorErr
|
||||||
@@ -120,8 +120,8 @@ func TestBuildWrapsConstructorError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsNilAdapter(t *testing.T) {
|
func TestInputAdapterRegistryBuildRejectsNilAdapter(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -138,9 +138,9 @@ func TestBuildRejectsNilAdapter(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
func TestInputAdapterRegistryBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
if err := registry.Register("generic-input", fakeConstructor("other-input")); err != nil {
|
if err := registry.Register("generic-input", fakeInputAdapterConstructor("other-input")); err != nil {
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -154,10 +154,10 @@ func TestBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
func TestInputAdapterRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
for _, key := range []string{"zeta", "alpha", "middle"} {
|
for _, key := range []string{"zeta", "alpha", "middle"} {
|
||||||
if err := registry.Register(key, fakeConstructor(key)); err != nil {
|
if err := registry.Register(key, fakeInputAdapterConstructor(key)); err != nil {
|
||||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -175,10 +175,10 @@ func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNilRegistryBehavior(t *testing.T) {
|
func TestInputAdapterRegistryNilRegistryBehavior(t *testing.T) {
|
||||||
var registry *Registry
|
var registry *InputAdapterRegistry
|
||||||
|
|
||||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err == nil {
|
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err == nil {
|
||||||
t.Fatal("Register() error = nil, want error")
|
t.Fatal("Register() error = nil, want error")
|
||||||
}
|
}
|
||||||
if _, err := registry.Build("generic-input"); err == nil {
|
if _, err := registry.Build("generic-input"); err == nil {
|
||||||
@@ -189,8 +189,8 @@ func TestNilRegistryBehavior(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildRejectsEmptyKey(t *testing.T) {
|
func TestInputAdapterRegistryBuildRejectsEmptyKey(t *testing.T) {
|
||||||
registry := New()
|
registry := NewInputAdapterRegistry()
|
||||||
|
|
||||||
_, err := registry.Build(" \n")
|
_, err := registry.Build(" \n")
|
||||||
|
|
||||||
@@ -206,7 +206,7 @@ type fakeAdapter struct {
|
|||||||
key string
|
key string
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeConstructor(key string) Constructor {
|
func fakeInputAdapterConstructor(key string) InputAdapterConstructor {
|
||||||
return func() (contracts.InputAdapter, error) {
|
return func() (contracts.InputAdapter, error) {
|
||||||
return fakeAdapter{key: key}, nil
|
return fakeAdapter{key: key}, nil
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package runner
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -8,14 +8,14 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/extractorregistry"
|
|
||||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
||||||
var builtKeys []string
|
var builtKeys []string
|
||||||
var executedKeys []string
|
var executedKeys []string
|
||||||
registry := extractorregistry.New()
|
registry := NewExtractorRegistry()
|
||||||
|
|
||||||
registerIntegrationExtractor(t, registry, "second", &builtKeys, &executedKeys, []contracts.Validator{
|
registerIntegrationExtractor(t, registry, "second", &builtKeys, &executedKeys, []contracts.Validator{
|
||||||
integrationValidator{name: "reject-second", approve: false},
|
integrationValidator{name: "reject-second", approve: false},
|
||||||
@@ -46,7 +46,7 @@ func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func registerIntegrationExtractor(t *testing.T, registry *extractorregistry.Registry, key string, builtKeys *[]string, executedKeys *[]string, validators []contracts.Validator) {
|
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, builtKeys *[]string, executedKeys *[]string, validators []contracts.Validator) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
if err := registry.Register(key, func() (contracts.Extractor, error) {
|
if err := registry.Register(key, func() (contracts.Extractor, error) {
|
||||||
@@ -82,7 +82,7 @@ func (extractor integrationExtractor) Validators() []contracts.Validator {
|
|||||||
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||||
*extractor.executedKeys = append(*extractor.executedKeys, extractor.key)
|
*extractor.executedKeys = append(*extractor.executedKeys, extractor.key)
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{Payload: []byte(`{"value":true}`)},
|
{Payload: []byte(`{"value":true}`)},
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -101,9 +101,9 @@ func (validator integrationValidator) Validate(ctx context.Context, req contract
|
|||||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
||||||
for _, candidate := range req.Candidates {
|
for _, candidate := range req.Candidates {
|
||||||
if validator.approve {
|
if validator.approve {
|
||||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||||
} else {
|
} else {
|
||||||
decisions = append(decisions, validationhelpers.Rejected(candidate.Index, "invalid", "not accepted"))
|
decisions = append(decisions, validate.Rejected(candidate.Index, "invalid", "not accepted"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return contracts.ValidationResult{
|
return contracts.ValidationResult{
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package runner
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ExtractorFactory interface {
|
type ExtractorFactory interface {
|
||||||
@@ -90,8 +90,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.Candidate, nextIndex *int) ([]artifacts.Candidate, error) {
|
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate, nextIndex *int) ([]artifacts.ArtifactCandidate, error) {
|
||||||
normalized := make([]artifacts.Candidate, 0, len(candidates))
|
normalized := make([]artifacts.ArtifactCandidate, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
candidate.Index = *nextIndex
|
candidate.Index = *nextIndex
|
||||||
*nextIndex = *nextIndex + 1
|
*nextIndex = *nextIndex + 1
|
||||||
@@ -119,7 +119,7 @@ func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.C
|
|||||||
return normalized, nil
|
return normalized, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runValidators(ctx context.Context, extractor contracts.Extractor, doc *source.SourceDocument, candidates []artifacts.Candidate, metadata map[string]any) ([]artifacts.Candidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
|
func runValidators(ctx context.Context, extractor contracts.Extractor, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
|
||||||
eligible := candidates
|
eligible := candidates
|
||||||
var rejected []artifacts.RejectedArtifact
|
var rejected []artifacts.RejectedArtifact
|
||||||
var warnings []contracts.Warning
|
var warnings []contracts.Warning
|
||||||
@@ -140,7 +140,7 @@ func runValidators(ctx context.Context, extractor contracts.Extractor, doc *sour
|
|||||||
if result.ValidatorName != validator.Name() {
|
if result.ValidatorName != validator.Name() {
|
||||||
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
|
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
|
||||||
}
|
}
|
||||||
if err := validators.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
|
if err := validate.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
|
||||||
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractor.Key(), validator.Name(), err)
|
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractor.Key(), validator.Name(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -149,7 +149,7 @@ func runValidators(ctx context.Context, extractor contracts.Extractor, doc *sour
|
|||||||
decisions[decision.CandidateIndex] = decision
|
decisions[decision.CandidateIndex] = decision
|
||||||
}
|
}
|
||||||
|
|
||||||
nextEligible := make([]artifacts.Candidate, 0, len(eligible))
|
nextEligible := make([]artifacts.ArtifactCandidate, 0, len(eligible))
|
||||||
for _, candidate := range eligible {
|
for _, candidate := range eligible {
|
||||||
decision := decisions[candidate.Index]
|
decision := decisions[candidate.Index]
|
||||||
if decision.Approved {
|
if decision.Approved {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package runner
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewAndDataTypes(t *testing.T) {
|
func TestNewAndDataTypes(t *testing.T) {
|
||||||
@@ -99,11 +99,11 @@ func TestRunRejectsNilExtractorFromFactory(t *testing.T) {
|
|||||||
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
||||||
var order []string
|
var order []string
|
||||||
var seenIndices []int
|
var seenIndices []int
|
||||||
recordIndices := func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
recordIndices := func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
seenIndices = append(seenIndices, candidate.Index)
|
seenIndices = append(seenIndices, candidate.Index)
|
||||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||||
}
|
}
|
||||||
return decisions
|
return decisions
|
||||||
}
|
}
|
||||||
@@ -133,7 +133,7 @@ func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{{Payload: []byte(`{"value":true}`)}}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.ArtifactCandidate{{Payload: []byte(`{"value":true}`)}}},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||||
@@ -150,18 +150,18 @@ func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
|||||||
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
candidate artifacts.Candidate
|
candidate artifacts.ArtifactCandidate
|
||||||
error string
|
error string
|
||||||
}{
|
}{
|
||||||
{name: "extractor key", candidate: artifacts.Candidate{ExtractorKey: "other"}, error: "extractor_key"},
|
{name: "extractor key", candidate: artifacts.ArtifactCandidate{ExtractorKey: "other"}, error: "extractor_key"},
|
||||||
{name: "artifact type", candidate: artifacts.Candidate{ArtifactType: "other"}, error: "artifact_type"},
|
{name: "artifact type", candidate: artifacts.ArtifactCandidate{ArtifactType: "other"}, error: "artifact_type"},
|
||||||
{name: "schema version", candidate: artifacts.Candidate{SchemaVersion: "other"}, error: "schema_version"},
|
{name: "schema version", candidate: artifacts.ArtifactCandidate{SchemaVersion: "other"}, error: "schema_version"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{tt.candidate}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.ArtifactCandidate{tt.candidate}},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||||
@@ -194,8 +194,8 @@ func TestRunApprovesCandidatesWithoutValidators(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
||||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
return []contracts.ValidationDecision{validate.Approved(candidates[0].Index)}
|
||||||
}}
|
}}
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{validator}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{validator}},
|
||||||
@@ -212,15 +212,15 @@ func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
||||||
var laterSeen int
|
var laterSeen int
|
||||||
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return []contracts.ValidationDecision{
|
return []contracts.ValidationDecision{
|
||||||
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
validate.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
||||||
validationhelpers.Approved(candidates[1].Index),
|
validate.Approved(candidates[1].Index),
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
laterSeen = len(candidates)
|
laterSeen = len(candidates)
|
||||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
return []contracts.ValidationDecision{validate.Approved(candidates[0].Index)}
|
||||||
}}
|
}}
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{rejectFirst, approveRemaining}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{rejectFirst, approveRemaining}},
|
||||||
@@ -254,7 +254,7 @@ func TestRunSurfacesValidatorNameMismatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
||||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return nil
|
return nil
|
||||||
}}
|
}}
|
||||||
factory := factoryWithValidator(validator)
|
factory := factoryWithValidator(validator)
|
||||||
@@ -359,7 +359,7 @@ type fakeExtractor struct {
|
|||||||
artifactType string
|
artifactType string
|
||||||
schemaVersion string
|
schemaVersion string
|
||||||
candidateCount int
|
candidateCount int
|
||||||
candidates []artifacts.Candidate
|
candidates []artifacts.ArtifactCandidate
|
||||||
validators []contracts.Validator
|
validators []contracts.Validator
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
@@ -386,9 +386,9 @@ func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.Extrac
|
|||||||
if extractor.order != nil {
|
if extractor.order != nil {
|
||||||
*extractor.order = append(*extractor.order, extractor.key)
|
*extractor.order = append(*extractor.order, extractor.key)
|
||||||
}
|
}
|
||||||
candidates := append([]artifacts.Candidate(nil), extractor.candidates...)
|
candidates := append([]artifacts.ArtifactCandidate(nil), extractor.candidates...)
|
||||||
for len(candidates) < extractor.candidateCount {
|
for len(candidates) < extractor.candidateCount {
|
||||||
candidates = append(candidates, artifacts.Candidate{Payload: []byte(`{"value":true}`)})
|
candidates = append(candidates, artifacts.ArtifactCandidate{Payload: []byte(`{"value":true}`)})
|
||||||
}
|
}
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: candidates,
|
Candidates: candidates,
|
||||||
@@ -399,7 +399,7 @@ func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.Extrac
|
|||||||
type fakeValidator struct {
|
type fakeValidator struct {
|
||||||
name string
|
name string
|
||||||
resultName string
|
resultName string
|
||||||
decisions func([]artifacts.Candidate) []contracts.ValidationDecision
|
decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
@@ -436,10 +436,10 @@ func factoryWithValidator(validator contracts.Validator) fakeFactory {
|
|||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func approveAll(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||||
}
|
}
|
||||||
return decisions
|
return decisions
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package validators
|
package validate
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -30,7 +30,7 @@ func Rejected(candidateIndex int, reasonCode string, message string) contracts.V
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []contracts.ValidationDecision) error {
|
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error {
|
||||||
if len(candidates) != len(decisions) {
|
if len(candidates) != len(decisions) {
|
||||||
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
|
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package validators
|
package validate
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
@@ -43,7 +43,7 @@ func TestRejectedTrimsReasonAndMessage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
|
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
|
||||||
candidates := []artifacts.Candidate{{Index: 4}, {Index: 8}}
|
candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}}
|
||||||
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
|
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
|
||||||
|
|
||||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
||||||
@@ -59,7 +59,7 @@ func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}},
|
[]artifacts.ArtifactCandidate{{Index: 1}},
|
||||||
[]contracts.ValidationDecision{Approved(2)},
|
[]contracts.ValidationDecision{Approved(2)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -68,7 +68,7 @@ func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 2}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -77,7 +77,7 @@ func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 2}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
||||||
[]contracts.ValidationDecision{Approved(1)},
|
[]contracts.ValidationDecision{Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,7 +86,7 @@ func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 1}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}},
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
Reference in New Issue
Block a user