Consolidate the architecture plan into fewer packages
This commit is contained in:
@@ -35,6 +35,10 @@ core source, runner, or LLM framework packages.
|
||||
|
||||
Extracted facts should be grounded with source references. Source references
|
||||
should point to generic source units, not to transcript-only structures.
|
||||
Artifact records should require source references by default unless their schema
|
||||
explicitly opts into ungrounded fields. Generic pipeline code should preserve
|
||||
source-reference ranges exactly and should not merge or rewrite overlapping
|
||||
ranges.
|
||||
|
||||
The application workflow is:
|
||||
|
||||
@@ -70,30 +74,17 @@ CLI and executable entrypoint:
|
||||
|
||||
Core deterministic model and policy:
|
||||
|
||||
- `internal/core/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||
- `internal/core/source`: source document, source unit, and source reference types.
|
||||
- `internal/core/sourcechunking`: deterministic chunking of ordered source units.
|
||||
- `internal/core/artifacts`: artifact envelope, artifact candidates, rejected artifacts, and manifests.
|
||||
- `internal/core/diagnostics`: run directories and diagnostics artifact paths.
|
||||
- `internal/core/reporting`: process reports and report serialization.
|
||||
- `internal/core/inputcatalog`: known input adapter keys and metadata.
|
||||
- `internal/core/extractorcatalog`: known extractor keys and metadata.
|
||||
- `internal/core/config`: configuration structs, defaults, loading, precedence, and validation, once config exists.
|
||||
|
||||
Reusable framework plumbing:
|
||||
|
||||
- `internal/framework/contracts`: core interfaces and transport-neutral request/response contracts.
|
||||
- `internal/framework/runner`: orchestration across adapters, extractors, validators, and artifact output.
|
||||
- `internal/framework/pipeline`: shared pipeline-stage orchestration types, when needed.
|
||||
- `internal/framework/extraction`: shared extraction helper code.
|
||||
- `internal/framework/merge`: shared merge-stage behavior.
|
||||
- `internal/framework/normalize`: shared normalization-stage behavior.
|
||||
- `internal/framework/output`: output encoding contracts and shared helpers.
|
||||
- `internal/framework/validators`: shared validator runtime behavior and decision checks.
|
||||
- `internal/framework/llm`: LLM runtime, scheduling, and provider adapters.
|
||||
- `internal/framework/responseschema`: embedded structured-output schema registry.
|
||||
- `internal/framework/structuredoutput`: structured-output parsing and malformed-response handling.
|
||||
- `internal/framework/promptcontext`: source-document prompt rendering helpers.
|
||||
- `internal/framework/warnings`: shared warning records.
|
||||
- `internal/framework/pipeline`: runner, pipeline-stage orchestration, registries, and small shared stage helpers.
|
||||
- `internal/framework/validate`: shared validator runtime behavior and decision checks.
|
||||
- `internal/framework/llm`: LLM clients, scheduling, structured-output parsing, and response-schema registry, once LLM runtime exists.
|
||||
- `internal/framework/prompt`: embedded prompt assets, prompt registry, and prompt rendering helpers, once prompt assets exist.
|
||||
|
||||
Domain implementations:
|
||||
|
||||
@@ -104,12 +95,17 @@ Domain implementations:
|
||||
- `internal/modules/normalize/<name>` or `internal/modules/normalize/<domain>/<name>`: normalize-stage modules.
|
||||
- `internal/modules/output/<name>`: output-stage modules.
|
||||
- `internal/validators/<validator>`: built-in validator implementations.
|
||||
- `internal/prompts`: embedded prompt assets and prompt metadata registry.
|
||||
- `internal/transport/http`: shared HTTP client code, if needed by provider integrations.
|
||||
|
||||
Package-private implementation constants may live near the package that owns
|
||||
them, preferably in `constants.go` when useful.
|
||||
|
||||
Start with fewer, larger framework packages. Split a package only when a real
|
||||
boundary proves itself through import direction, ownership, test seams, or
|
||||
substantial file size. Do not create catalog, diagnostics, reporting,
|
||||
structured-output, response-schema, output, merge, normalize, extraction, or
|
||||
warnings packages merely because the concepts exist in the architecture.
|
||||
|
||||
## Stage Modules
|
||||
|
||||
Concrete business logic should live under `internal/modules/<stage>/...`.
|
||||
@@ -143,10 +139,14 @@ Other packages should interact with source input through adapter contracts and
|
||||
core source types. Input module implementation details and external dependency
|
||||
types must not leak into framework or extract module packages.
|
||||
|
||||
Input module metadata may preserve source-specific facts such as transcript speaker,
|
||||
timestamps, Markdown heading path, page number, or block ID. Framework code may
|
||||
carry metadata through, but should not require a specific adapter's metadata
|
||||
shape.
|
||||
Input module metadata may preserve source-specific facts such as transcript
|
||||
speaker, timestamps, Markdown heading path, page number, or block ID. Framework
|
||||
code may carry metadata through, but should not require a specific adapter's
|
||||
metadata shape.
|
||||
|
||||
Core source metadata should remain `map[string]any`. Document well-known keys
|
||||
as conventions, and let input modules expose typed accessor helpers for their
|
||||
own metadata when useful.
|
||||
|
||||
## Extractors
|
||||
|
||||
@@ -163,6 +163,11 @@ Each extract module owns:
|
||||
Extract modules should depend on framework contracts and core source/artifact
|
||||
types. They should not depend on concrete input module packages.
|
||||
|
||||
Extraction requests should carry the active source chunk plus optional ambient
|
||||
context, such as a document synopsis, prior-chunk summaries, known entities, or
|
||||
other module-provided state. The context may be empty for simple modules, but
|
||||
the contract should not assume extraction is always chunk-local.
|
||||
|
||||
Extractors should not be the only place where chunking, merging, or
|
||||
normalization happens. They may choose processing mode or provide domain-specific
|
||||
merge/normalization behavior when generic behavior is insufficient, but the
|
||||
@@ -200,6 +205,15 @@ The framework should allow serial and parallel chunk processing. The first
|
||||
implementation may execute chunks serially for determinism, but contracts should
|
||||
not prevent later parallel execution.
|
||||
|
||||
Final durable output should use one artifact file per artifact type plus a
|
||||
run-level manifest/index file. Framework artifact flow should use a generic
|
||||
envelope with `json.RawMessage` payloads; extract modules should own typed Go
|
||||
structs at their boundaries and encode into that generic envelope before
|
||||
returning to framework code.
|
||||
|
||||
Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||
format version.
|
||||
|
||||
## Validators
|
||||
|
||||
Validators should be independently testable and composable.
|
||||
@@ -209,7 +223,11 @@ present. Validator decision semantics should be explicit: each candidate
|
||||
artifact should receive exactly one decision from each validator that evaluates
|
||||
it.
|
||||
|
||||
Shared validator runtime mechanics belong under `internal/framework/validators`.
|
||||
LLM-backed review belongs in module-owned validator chains, not in a separate
|
||||
global review phase. Extract modules and normalize modules may both use
|
||||
deterministic and LLM-backed validators.
|
||||
|
||||
Shared validator runtime mechanics belong under `internal/framework/validate`.
|
||||
Concrete validator behavior belongs under `internal/validators/<validator>`.
|
||||
|
||||
## LLM Runtime
|
||||
@@ -296,6 +314,12 @@ and LLM clients where practical.
|
||||
Contract-first work should include fake implementations that prove interfaces
|
||||
compose before real input modules or extract modules depend on them.
|
||||
|
||||
Once the pipeline-stage contracts exist, maintain a fixture-driven walking
|
||||
skeleton that exercises input, chunk, extract, merge, normalize, and output
|
||||
stages with fake modules and fake external clients. This test should protect
|
||||
stage composition continuously while real modules are introduced over later
|
||||
checkpoints.
|
||||
|
||||
Config examples should be load-tested once config files exist. Important CLI
|
||||
workflows should have parser or command tests. Adapter, extractor, and validator
|
||||
contracts should have focused tests that do not require running the full
|
||||
|
||||
@@ -38,13 +38,11 @@ Out of scope:
|
||||
|
||||
The repository should contain a minimal framework composition layer:
|
||||
|
||||
- `internal/framework/inputregistry` registers and builds input adapter
|
||||
constructors by stable key.
|
||||
- `internal/framework/extractorregistry` registers and builds extractor
|
||||
constructors by stable key.
|
||||
- `internal/framework/validators` provides shared validator decision helpers and
|
||||
- `internal/framework/pipeline` registers and builds input adapter constructors
|
||||
and extractor constructors by stable key.
|
||||
- `internal/framework/validate` provides shared validator decision helpers and
|
||||
cardinality checks.
|
||||
- `internal/framework/runner` executes configured extractors against a
|
||||
- `internal/framework/pipeline` executes configured extractors against a
|
||||
`SourceDocument`, applies validator chains, and returns approved and rejected
|
||||
artifacts.
|
||||
|
||||
|
||||
@@ -16,9 +16,11 @@ input -> chunk -> extract -> merge -> normalize -> output
|
||||
```
|
||||
|
||||
Checkpoint 3 should define the contracts and minimal fake-tested framework
|
||||
behavior for chunking, per-chunk processing, merging, and normalization. It
|
||||
should not add real input modules, real domain extract modules, LLM provider code,
|
||||
or output encoders.
|
||||
behavior for chunking, per-chunk extraction, merging, and normalization. It
|
||||
should also introduce a fixture-driven walking skeleton that exercises the full
|
||||
stage sequence with fake modules and a fake LLM client. It should not add real
|
||||
input modules, real domain extract modules, LLM provider code, or production
|
||||
output modules.
|
||||
|
||||
## Scope
|
||||
|
||||
@@ -29,10 +31,13 @@ In scope:
|
||||
- extract-stage contract for extractors operating on chunks;
|
||||
- merge-stage contract;
|
||||
- normalize-stage contract;
|
||||
- output-stage contract shape, if useful for pipeline completeness;
|
||||
- output-stage contract and fake output encoder for pipeline completeness;
|
||||
- runner/pipeline updates that exercise these stages with fake components;
|
||||
- generic append/chronological merge behavior for artifact candidates when
|
||||
appropriate.
|
||||
appropriate;
|
||||
- fixture-driven walking skeleton test for
|
||||
`input -> chunk -> extract -> merge -> normalize -> output`;
|
||||
- fake `StructuredLLMClient` wired through a trivial extractor.
|
||||
|
||||
Out of scope:
|
||||
|
||||
@@ -42,7 +47,8 @@ Out of scope:
|
||||
- prompt assets;
|
||||
- response schema assets;
|
||||
- diagnostics run directory;
|
||||
- production output serialization.
|
||||
- real CLI command behavior;
|
||||
- production output serialization or durable output writing.
|
||||
|
||||
## Target End State
|
||||
|
||||
@@ -53,12 +59,23 @@ The repository should contain explicit pipeline-stage contracts:
|
||||
- `Extractor`: `SourceChunk` to artifact candidates.
|
||||
- `Merger`: per-chunk candidates to merged candidates.
|
||||
- `Normalizer`: merged candidates to normalized candidates.
|
||||
- `OutputEncoder`: final artifact bundle to bytes, if introduced in this
|
||||
checkpoint.
|
||||
- `OutputEncoder`: final artifact bundle to bytes.
|
||||
|
||||
The runner should orchestrate fake implementations through chunk, extract,
|
||||
merge, normalize, and approval/validation behavior in tests.
|
||||
|
||||
The checkpoint should include one fixture-driven integration test that starts
|
||||
from fixture input bytes and ends at encoded output bytes. The fixture should
|
||||
use a fake input adapter, deterministic chunker, trivial extractor, fake
|
||||
structured LLM client, generic merger, no-op normalizer, and fake or minimal
|
||||
JSON output encoder. This is a contract exercise, not a useful user-facing
|
||||
workflow.
|
||||
|
||||
The walking skeleton should live in `internal/framework/pipeline`, with
|
||||
fixtures under that package's `testdata/`. If the CLI has an extract command by
|
||||
then, the same fixture may also be exercised through CLI wiring; if not, CLI
|
||||
coverage remains deferred.
|
||||
|
||||
## Design Intent
|
||||
|
||||
Chunking is a core application concern because many source documents, especially
|
||||
@@ -109,17 +126,39 @@ Domain-specific normalizers may later:
|
||||
- enforce chronological or source-reference consistency;
|
||||
- attach normalization warnings.
|
||||
|
||||
## Walking Skeleton Fixture
|
||||
|
||||
The fixture-driven skeleton should prove the staged architecture continuously as
|
||||
new contracts are added. It should be deliberately small:
|
||||
|
||||
- a fixture source document or raw source input with two or three source units;
|
||||
- a fake input adapter that parses the fixture into `SourceDocument`;
|
||||
- a deterministic chunker that produces multiple chunks;
|
||||
- a trivial extractor that calls the fake structured LLM client and emits one
|
||||
artifact candidate per chunk;
|
||||
- a generic append-in-chunk-order merger;
|
||||
- a no-op normalizer;
|
||||
- a fake or minimal JSON output encoder;
|
||||
- an expected output fixture checked byte-for-byte or by unmarshaled structural
|
||||
equality.
|
||||
|
||||
The fake LLM client should be part of the test setup so the contract is
|
||||
exercised without introducing provider code, prompt assets, or response schema
|
||||
assets.
|
||||
|
||||
## Done Criteria
|
||||
|
||||
- `go test ./...` passes.
|
||||
- Pipeline-stage contracts are explicit and source/domain agnostic.
|
||||
- Fake tests prove input source documents can be chunked, extracted, merged, and
|
||||
normalized.
|
||||
- A fixture-driven walking skeleton proves fake input, chunk, extract, merge,
|
||||
normalize, and output modules compose end to end with a fake LLM client.
|
||||
- Merge and normalize are distinct concepts in code and tests.
|
||||
- The runner no longer implies whole-document-only extraction as the core
|
||||
application model.
|
||||
- No concrete input module, extract module, LLM provider, prompt, response schema,
|
||||
diagnostics, config, or D&D artifact code is added.
|
||||
- No concrete input module, domain extract module, LLM provider, prompt,
|
||||
response schema, diagnostics, config, or D&D artifact code is added.
|
||||
|
||||
## Review Questions
|
||||
|
||||
|
||||
@@ -64,26 +64,26 @@ Add a constructor-based registry for `contracts.InputAdapter`.
|
||||
|
||||
### Files To Add
|
||||
|
||||
- `internal/framework/inputregistry/registry.go`
|
||||
- `internal/framework/inputregistry/registry_test.go`
|
||||
- `internal/framework/pipeline/input_registry.go`
|
||||
- `internal/framework/pipeline/input_registry_test.go`
|
||||
|
||||
### Required API
|
||||
|
||||
Create package `inputregistry`.
|
||||
Extend package `pipeline`.
|
||||
|
||||
Define:
|
||||
|
||||
```go
|
||||
type Constructor func() (contracts.InputAdapter, error)
|
||||
type InputAdapterConstructor func() (contracts.InputAdapter, error)
|
||||
|
||||
type Registry struct {
|
||||
type InputAdapterRegistry struct {
|
||||
// unexported fields
|
||||
}
|
||||
|
||||
func New() *Registry
|
||||
func (r *Registry) Register(key string, constructor Constructor) error
|
||||
func (r *Registry) Build(key string) (contracts.InputAdapter, error)
|
||||
func (r *Registry) RegisteredKeys() []string
|
||||
func NewInputAdapterRegistry() *InputAdapterRegistry
|
||||
func (r *InputAdapterRegistry) Register(key string, constructor InputAdapterConstructor) error
|
||||
func (r *InputAdapterRegistry) Build(key string) (contracts.InputAdapter, error)
|
||||
func (r *InputAdapterRegistry) RegisteredKeys() []string
|
||||
```
|
||||
|
||||
### Required Behavior
|
||||
@@ -137,8 +137,8 @@ Use fake adapters only. Do not add concrete input module packages.
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/inputregistry
|
||||
go test ./internal/framework/inputregistry
|
||||
gofmt -w internal/framework/pipeline
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./...
|
||||
```
|
||||
|
||||
@@ -150,31 +150,31 @@ Add a constructor-based registry for `contracts.Extractor`.
|
||||
|
||||
### Files To Add
|
||||
|
||||
- `internal/framework/extractorregistry/registry.go`
|
||||
- `internal/framework/extractorregistry/registry_test.go`
|
||||
- `internal/framework/pipeline/extractor_registry.go`
|
||||
- `internal/framework/pipeline/extractor_registry_test.go`
|
||||
|
||||
### Required API
|
||||
|
||||
Create package `extractorregistry`.
|
||||
Extend package `pipeline`.
|
||||
|
||||
Define:
|
||||
|
||||
```go
|
||||
type Constructor func() (contracts.Extractor, error)
|
||||
type ExtractorConstructor func() (contracts.Extractor, error)
|
||||
|
||||
type Registry struct {
|
||||
type ExtractorRegistry struct {
|
||||
// unexported fields
|
||||
}
|
||||
|
||||
func New() *Registry
|
||||
func (r *Registry) Register(key string, constructor Constructor) error
|
||||
func (r *Registry) Build(key string) (contracts.Extractor, error)
|
||||
func (r *Registry) RegisteredKeys() []string
|
||||
func NewExtractorRegistry() *ExtractorRegistry
|
||||
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error
|
||||
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error)
|
||||
func (r *ExtractorRegistry) RegisteredKeys() []string
|
||||
```
|
||||
|
||||
### Required Behavior
|
||||
|
||||
Mirror `inputregistry` behavior, but for `contracts.Extractor`.
|
||||
Mirror input adapter registry behavior, but for `contracts.Extractor`.
|
||||
|
||||
`Build` must reject extractors whose `Key()` does not match the normalized key.
|
||||
It should not validate artifact type, schema version, or validator chain. Those
|
||||
@@ -203,8 +203,8 @@ Use fake extractors only. Do not add concrete extract module packages.
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/extractorregistry
|
||||
go test ./internal/framework/extractorregistry
|
||||
gofmt -w internal/framework/pipeline
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./...
|
||||
```
|
||||
|
||||
@@ -216,12 +216,12 @@ Add shared validator decision helpers and decision-cardinality enforcement.
|
||||
|
||||
### Files To Add
|
||||
|
||||
- `internal/framework/validators/validators.go`
|
||||
- `internal/framework/validators/validators_test.go`
|
||||
- `internal/framework/validate/validate.go`
|
||||
- `internal/framework/validate/validate_test.go`
|
||||
|
||||
### Required API
|
||||
|
||||
Create package `validators`.
|
||||
Create package `validate`.
|
||||
|
||||
Define reason constants:
|
||||
|
||||
@@ -285,8 +285,8 @@ Cover:
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/validators
|
||||
go test ./internal/framework/validators
|
||||
gofmt -w internal/framework/validate
|
||||
go test ./internal/framework/validate
|
||||
go test ./...
|
||||
```
|
||||
|
||||
@@ -294,17 +294,17 @@ go test ./...
|
||||
|
||||
### Goal
|
||||
|
||||
Add runner package types and constructor without implementing the full run loop
|
||||
Add pipeline runner types and constructor without implementing the full run loop
|
||||
yet.
|
||||
|
||||
### Files To Add
|
||||
|
||||
- `internal/framework/runner/runner.go`
|
||||
- `internal/framework/runner/runner_test.go`
|
||||
- `internal/framework/pipeline/runner.go`
|
||||
- `internal/framework/pipeline/runner_test.go`
|
||||
|
||||
### Required API
|
||||
|
||||
Create package `runner`.
|
||||
Create or extend package `pipeline`.
|
||||
|
||||
Define:
|
||||
|
||||
@@ -364,8 +364,8 @@ assertions. Runner should depend only on framework contracts and core packages.
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/runner
|
||||
go test ./internal/framework/runner
|
||||
gofmt -w internal/framework/pipeline
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./...
|
||||
```
|
||||
|
||||
@@ -378,8 +378,8 @@ artifacts.
|
||||
|
||||
### Files To Update
|
||||
|
||||
- `internal/framework/runner/runner.go`
|
||||
- `internal/framework/runner/runner_test.go`
|
||||
- `internal/framework/pipeline/runner.go`
|
||||
- `internal/framework/pipeline/runner_test.go`
|
||||
|
||||
### Required API
|
||||
|
||||
@@ -423,7 +423,7 @@ Validation behavior:
|
||||
- if an extractor has no validators, all normalized candidates are approved;
|
||||
- each validator receives only currently eligible candidates;
|
||||
- each validator result must have `ValidatorName == Validator.Name()`;
|
||||
- call `validators.EnforceDecisionCardinality` for every validator result;
|
||||
- call `validate.EnforceDecisionCardinality` for every validator result;
|
||||
- approved decisions keep candidates eligible for the next validator;
|
||||
- rejected decisions create `artifacts.RejectedArtifact` records using the
|
||||
candidate, validator name, reason code, and message;
|
||||
@@ -469,8 +469,8 @@ Use fake extractors, validators, and factories only.
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/runner
|
||||
go test ./internal/framework/runner
|
||||
gofmt -w internal/framework/pipeline
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./...
|
||||
```
|
||||
|
||||
@@ -478,16 +478,16 @@ go test ./...
|
||||
|
||||
### Goal
|
||||
|
||||
Prove the extractor registry and runner compose without adding real process
|
||||
Prove the extractor registry and runner compose without adding real extract
|
||||
modules.
|
||||
|
||||
### Files To Add
|
||||
|
||||
- `internal/framework/runner/registry_integration_test.go`
|
||||
- `internal/framework/pipeline/registry_integration_test.go`
|
||||
|
||||
### Required Test Scenario
|
||||
|
||||
Use `extractorregistry.New()` to register two fake extractor constructors.
|
||||
Use `NewExtractorRegistry()` to register two fake extractor constructors.
|
||||
|
||||
Run the runner with:
|
||||
|
||||
@@ -509,8 +509,8 @@ Assertions:
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/framework/runner
|
||||
go test ./internal/framework/runner
|
||||
gofmt -w internal/framework/pipeline
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./...
|
||||
go build ./cmd/notarius
|
||||
```
|
||||
|
||||
@@ -22,7 +22,7 @@ in core framework packages.
|
||||
|
||||
The application should follow the same broad architecture as Audita:
|
||||
|
||||
- deterministic core packages for config, source documents, artifacts, diagnostics, and reporting;
|
||||
- deterministic core packages for source documents, artifacts, and configuration once needed;
|
||||
- input-stage modules that translate external source formats into a small internal source model;
|
||||
- reusable framework packages for contracts, orchestration, LLM runtime, structured output, and validation;
|
||||
- independent extract-stage modules that own domain-specific behavior;
|
||||
@@ -48,28 +48,14 @@ artifacts rather than proposing and applying transcript corrections.
|
||||
cmd/notarius
|
||||
internal/cli
|
||||
|
||||
internal/core/config
|
||||
internal/core/source
|
||||
internal/core/sourcechunking
|
||||
internal/core/artifacts
|
||||
internal/core/diagnostics
|
||||
internal/core/reporting
|
||||
internal/core/extractorcatalog
|
||||
internal/core/inputcatalog
|
||||
|
||||
internal/framework/contracts
|
||||
internal/framework/extraction
|
||||
internal/framework/runner
|
||||
internal/framework/pipeline
|
||||
internal/framework/merge
|
||||
internal/framework/normalize
|
||||
internal/framework/output
|
||||
internal/framework/validators
|
||||
internal/framework/validate
|
||||
internal/framework/llm
|
||||
internal/framework/responseschema
|
||||
internal/framework/structuredoutput
|
||||
internal/framework/promptcontext
|
||||
internal/framework/warnings
|
||||
internal/framework/prompt
|
||||
|
||||
internal/modules/input/seriatim
|
||||
internal/modules/input/markdown
|
||||
@@ -95,7 +81,6 @@ internal/validators/schema_validity
|
||||
internal/validators/domain_consistency
|
||||
internal/validators/llm_review
|
||||
|
||||
internal/prompts
|
||||
examples
|
||||
docs/internal
|
||||
```
|
||||
@@ -104,6 +89,15 @@ The `markdown` input module and D&D-specific chunk, merge, normalize, and
|
||||
output modules are listed as likely future packages. The MVP should implement
|
||||
only the stage modules needed by the checkpoint sequence.
|
||||
|
||||
`internal/core/config` should be added when production configuration exists.
|
||||
|
||||
The framework package list is intentionally consolidated. `pipeline` should own
|
||||
runner orchestration, stage registries, and small merge/normalize/output helpers
|
||||
until those boundaries prove they need separate packages. `llm` should own
|
||||
structured output and response-schema mechanics until those concerns become too
|
||||
large or import-heavy. `prompt` should own prompt assets and rendering helpers
|
||||
once prompt assets exist.
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### SourceDocument
|
||||
@@ -138,6 +132,13 @@ Initial source-unit assumptions:
|
||||
- adapter-specific metadata may carry speaker, timestamps, heading paths, page
|
||||
numbers, or other source details.
|
||||
|
||||
Core source metadata should remain `map[string]any`. Notarius should not define
|
||||
a universal document model. Instead, the project should document well-known
|
||||
metadata keys, such as `speaker`, `start`, `end`, and `heading_path`, as
|
||||
conventions. Input modules may export typed accessor helpers for their own
|
||||
metadata, such as `seriatim.SpeakerOf(unit)`, without leaking those helpers into
|
||||
core framework contracts.
|
||||
|
||||
### Input Module / Adapter Contract
|
||||
|
||||
Hexagonal boundary for external source formats.
|
||||
@@ -183,6 +184,11 @@ Initial source-reference validation should require:
|
||||
Transcript-oriented output can still present these as transcript segment ranges
|
||||
when the adapter metadata makes that interpretation available.
|
||||
|
||||
Source references should preserve the exact ranges produced by extractors and
|
||||
validators. Overlapping ranges should not be merged or rewritten by generic
|
||||
pipeline code. If a domain module wants a derived compact range later, that
|
||||
should be additional output, not a replacement for the original evidence.
|
||||
|
||||
### Extractor
|
||||
|
||||
Reusable module contract for producing one artifact type.
|
||||
@@ -201,6 +207,13 @@ An extractor should receive either a whole source document or a source chunk,
|
||||
depending on processing mode. It should return typed artifact candidates plus
|
||||
warnings. It should not mutate the source document.
|
||||
|
||||
`ExtractionRequest` should be designed now to carry both the active chunk and
|
||||
optional ambient context, even if the MVP leaves that context empty. Useful
|
||||
ambient context may include a document synopsis, prior-chunk summaries, known
|
||||
entities, or other module-provided state. D&D spell extraction can likely work
|
||||
per chunk, but combat, NPC, and identity-oriented extraction will need broader
|
||||
context. Adding the field later would force churn across every extractor.
|
||||
|
||||
Extract modules own domain concepts. For example, D&D spell extraction should
|
||||
live under `internal/modules/extract/dnd/spells`; a future to-do extractor for
|
||||
notes should live under a different extract-module path and use the same
|
||||
@@ -247,14 +260,41 @@ Validators should cover:
|
||||
Validator output should follow Audita's decision-cardinality model: each
|
||||
candidate artifact receives exactly one decision per validator.
|
||||
|
||||
LLM-backed review should be modeled as part of a module's validator chain, not
|
||||
as a separate global review phase. Extract modules should be able to attach one
|
||||
or more deterministic or LLM-backed validators. Normalize-stage modules may also
|
||||
run validator chains, including LLM-backed validators, when semantic
|
||||
reconciliation needs review.
|
||||
|
||||
### Artifact
|
||||
|
||||
Final approved JSON output from one or more extractors.
|
||||
|
||||
Artifacts should preserve enough metadata to support downstream validation,
|
||||
debugging, and replay. The exact top-level envelope is still open, but should
|
||||
include artifact type, schema version, extracted records, source references, and
|
||||
run manifest data.
|
||||
debugging, and replay.
|
||||
|
||||
The pipeline should carry artifact candidates through a generic envelope with a
|
||||
`json.RawMessage` payload. Extract modules should own typed Go structs at their
|
||||
module boundary, then encode those typed records into the generic artifact
|
||||
candidate envelope before returning to framework code. This keeps stage
|
||||
contracts simple and avoids generic type plumbing across unrelated artifact
|
||||
families.
|
||||
|
||||
Final durable output should be one file per artifact type plus a run-level
|
||||
manifest/index file. This supports partial success and lets downstream consumers
|
||||
read only the artifact types they need. Each artifact file should include its
|
||||
artifact type, extractor key, extractor schema version, envelope format version,
|
||||
records, source references, and enough provenance to connect it to the run
|
||||
manifest.
|
||||
|
||||
Every artifact record should require source references unless that artifact
|
||||
schema explicitly opts into ungrounded fields. Artifact-level metadata, counts,
|
||||
run information, and other derived summary fields are exempt from the per-record
|
||||
grounding rule.
|
||||
|
||||
Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||
format version. A single global schema version would couple unrelated extractor
|
||||
release cadence.
|
||||
|
||||
### RunManifest
|
||||
|
||||
@@ -262,6 +302,7 @@ Per-run provenance record.
|
||||
|
||||
```go
|
||||
type RunManifest struct {
|
||||
EnvelopeVersion string `json:"envelope_version"`
|
||||
InputModule string `json:"input_module"`
|
||||
Chunker string `json:"chunker"`
|
||||
SourceDigests []string `json:"source_digests"`
|
||||
@@ -371,7 +412,7 @@ Proposed runner flow:
|
||||
7. Resolve the configured chunker.
|
||||
8. Chunk source units into deterministic source chunks.
|
||||
9. Resolve configured extractor instances through a registry.
|
||||
10. Process chunks in extractor-defined mode.
|
||||
10. Extract from chunks in extractor-defined mode.
|
||||
11. Merge per-chunk artifact candidates deterministically.
|
||||
12. Normalize merged artifact candidates.
|
||||
13. Run deterministic validators before LLM-backed validators.
|
||||
@@ -401,6 +442,11 @@ Reuse these architectural patterns:
|
||||
- validator decision cardinality and deterministic validator ordering;
|
||||
- CLI tests and fixture-driven integration tests.
|
||||
|
||||
The fixture-driven integration-test pattern should begin at checkpoint 3 with a
|
||||
walking skeleton over fake modules and a fake LLM client. Later checkpoints
|
||||
should replace fake pieces with real Seriatim, runtime, and D&D modules without
|
||||
losing that end-to-end contract coverage.
|
||||
|
||||
Avoid copying these Audita concepts directly:
|
||||
|
||||
- transcript-specific core types;
|
||||
@@ -426,32 +472,47 @@ compiling and targeted tests covering the newly introduced contracts or behavior
|
||||
5. [Seriatim Input Module](5-seriatim-input-module.md)
|
||||
6. [D&D Spells Extractor](6-dnd-spells-extractor.md)
|
||||
|
||||
The first useful vertical slice should arrive at checkpoint 6: Seriatim
|
||||
transcript input to validated D&D spell artifact output. Earlier checkpoints are
|
||||
intentionally contract-first and may not produce useful user output yet.
|
||||
The first contract-level walking skeleton should arrive at checkpoint 3: fixture
|
||||
input through fake input, chunk, extract, merge, normalize, and output modules
|
||||
with a fake LLM client. The first useful vertical slice should arrive at
|
||||
checkpoint 6: Seriatim transcript input to validated D&D spell artifact output.
|
||||
Earlier checkpoints remain contract-first and may not produce useful user output
|
||||
yet.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
- Final durable output should use one artifact file per artifact type plus a
|
||||
run-level manifest/index file.
|
||||
- Framework artifact flow should use a generic envelope with `json.RawMessage`
|
||||
payloads. Extract modules should use typed Go structs at their own boundaries.
|
||||
- Schemas should be versioned per extractor, with a separate envelope/manifest
|
||||
format version.
|
||||
- Artifact records should require source references by default. Individual
|
||||
schemas may explicitly opt into ungrounded fields. Artifact-level metadata is
|
||||
exempt.
|
||||
- Source-reference ranges should be preserved exactly. Generic pipeline code
|
||||
should not merge or rewrite overlapping ranges.
|
||||
- `ExtractionRequest` should carry the active chunk plus optional ambient
|
||||
context for document synopsis, prior-chunk summaries, known entities, or
|
||||
similar module-provided state.
|
||||
- LLM-backed review should be part of module-owned validator chains. Extract
|
||||
modules and normalize modules may both use deterministic and LLM-backed
|
||||
validators.
|
||||
- The Seriatim MVP should support only the minimal Seriatim schema. Broader
|
||||
Seriatim schema support should be added later without changing core source
|
||||
contracts.
|
||||
- Core source metadata should remain `map[string]any`. Well-known metadata keys
|
||||
should be documented as conventions, and input modules may expose typed
|
||||
accessor helpers for their own metadata.
|
||||
|
||||
## Open Design Questions
|
||||
|
||||
- Should final output be one combined artifact envelope or one file per
|
||||
extractor?
|
||||
- Should extractor output use typed Go structs per artifact or a generic
|
||||
artifact record with `json.RawMessage` payloads?
|
||||
- Should schemas be versioned per extractor, globally, or both?
|
||||
- Should every record require source references, or should some top-level
|
||||
artifact metadata be allowed without source references?
|
||||
- Should overlapping source-reference ranges be merged, preserved exactly, or
|
||||
both?
|
||||
- Should extraction run independently per source chunk only, or should some
|
||||
extractors receive whole-document context?
|
||||
- Which artifact types can use a generic append-in-chunk-order merger?
|
||||
- Which artifact types should use generic append-in-chunk-order merge, and which
|
||||
should use domain-specific merge?
|
||||
- Which artifact types need domain-specific normalization for deduplication,
|
||||
identity resolution, or consistency?
|
||||
- Should LLM review be part of each extractor's validator chain or a separate
|
||||
review phase?
|
||||
- Should the Seriatim adapter accept only its minimal schema initially or also
|
||||
support richer transcript schemas?
|
||||
- Should source-unit metadata be untyped `map[string]any`, typed extension
|
||||
structs, or both?
|
||||
- What should the configuration model look like for selecting input, chunk,
|
||||
extract, merge, normalize, output, and validator modules?
|
||||
|
||||
## Near-Term Documentation Tasks
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package extractorregistry
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -8,19 +8,19 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type Constructor func() (contracts.Extractor, error)
|
||||
type ExtractorConstructor func() (contracts.Extractor, error)
|
||||
|
||||
type Registry struct {
|
||||
constructors map[string]Constructor
|
||||
type ExtractorRegistry struct {
|
||||
constructors map[string]ExtractorConstructor
|
||||
}
|
||||
|
||||
func New() *Registry {
|
||||
return &Registry{
|
||||
constructors: make(map[string]Constructor),
|
||||
func NewExtractorRegistry() *ExtractorRegistry {
|
||||
return &ExtractorRegistry{
|
||||
constructors: make(map[string]ExtractorConstructor),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Register(key string, constructor Constructor) error {
|
||||
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
@@ -40,7 +40,7 @@ func (r *Registry) Register(key string, constructor Constructor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) Build(key string) (contracts.Extractor, error) {
|
||||
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func (r *Registry) Build(key string) (contracts.Extractor, error) {
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
func (r *Registry) RegisteredKeys() []string {
|
||||
func (r *ExtractorRegistry) RegisteredKeys() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package extractorregistry
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRegisterAndBuild(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err != nil {
|
||||
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -26,10 +26,10 @@ func TestRegisterAndBuild(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if err := registry.Register(" generic-extractor ", fakeConstructor("generic-extractor")); err != nil {
|
||||
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -42,10 +42,10 @@ func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsEmptyKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
err := registry.Register(" \t", fakeConstructor("generic-extractor"))
|
||||
err := registry.Register(" \t", fakeExtractorConstructor("generic-extractor"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
@@ -55,13 +55,13 @@ func TestRegisterRejectsEmptyKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
registry := New()
|
||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err != nil {
|
||||
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
err := registry.Register(" generic-extractor ", fakeConstructor("generic-extractor"))
|
||||
err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
@@ -71,8 +71,8 @@ func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsNilConstructor(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
err := registry.Register("generic-extractor", nil)
|
||||
|
||||
@@ -84,8 +84,8 @@ func TestRegisterRejectsNilConstructor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsUnknownKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
_, err := registry.Build("missing-extractor")
|
||||
|
||||
@@ -97,8 +97,8 @@ func TestBuildRejectsUnknownKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWrapsConstructorError(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
constructorErr := errors.New("constructor failed")
|
||||
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
||||
return nil, constructorErr
|
||||
@@ -119,8 +119,8 @@ func TestBuildWrapsConstructorError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsNilExtractor(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
|
||||
return nil, nil
|
||||
}); err != nil {
|
||||
@@ -137,9 +137,9 @@ func TestBuildRejectsNilExtractor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
||||
registry := New()
|
||||
if err := registry.Register("generic-extractor", fakeConstructor("other-extractor")); err != nil {
|
||||
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.Register("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -153,10 +153,10 @@ func TestBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
for _, key := range []string{"zeta", "alpha", "middle"} {
|
||||
if err := registry.Register(key, fakeConstructor(key)); err != nil {
|
||||
if err := registry.Register(key, fakeExtractorConstructor(key)); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
}
|
||||
@@ -174,10 +174,10 @@ func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilRegistryBehavior(t *testing.T) {
|
||||
var registry *Registry
|
||||
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
|
||||
var registry *ExtractorRegistry
|
||||
|
||||
if err := registry.Register("generic-extractor", fakeConstructor("generic-extractor")); err == nil {
|
||||
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if _, err := registry.Build("generic-extractor"); err == nil {
|
||||
@@ -188,8 +188,8 @@ func TestNilRegistryBehavior(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsEmptyKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
_, err := registry.Build(" \n")
|
||||
|
||||
@@ -201,32 +201,32 @@ func TestBuildRejectsEmptyKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type fakeExtractor struct {
|
||||
type registryFakeExtractor struct {
|
||||
key string
|
||||
}
|
||||
|
||||
func fakeConstructor(key string) Constructor {
|
||||
func fakeExtractorConstructor(key string) ExtractorConstructor {
|
||||
return func() (contracts.Extractor, error) {
|
||||
return fakeExtractor{key: key}, nil
|
||||
return registryFakeExtractor{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Key() string {
|
||||
func (extractor registryFakeExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) ArtifactType() string {
|
||||
func (extractor registryFakeExtractor) ArtifactType() string {
|
||||
return "generic-artifact"
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) SchemaVersion() string {
|
||||
func (extractor registryFakeExtractor) SchemaVersion() string {
|
||||
return "v1"
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Validators() []contracts.Validator {
|
||||
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package inputregistry
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -8,19 +8,19 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type Constructor func() (contracts.InputAdapter, error)
|
||||
type InputAdapterConstructor func() (contracts.InputAdapter, error)
|
||||
|
||||
type Registry struct {
|
||||
constructors map[string]Constructor
|
||||
type InputAdapterRegistry struct {
|
||||
constructors map[string]InputAdapterConstructor
|
||||
}
|
||||
|
||||
func New() *Registry {
|
||||
return &Registry{
|
||||
constructors: make(map[string]Constructor),
|
||||
func NewInputAdapterRegistry() *InputAdapterRegistry {
|
||||
return &InputAdapterRegistry{
|
||||
constructors: make(map[string]InputAdapterConstructor),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Register(key string, constructor Constructor) error {
|
||||
func (r *InputAdapterRegistry) Register(key string, constructor InputAdapterConstructor) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("input adapter registry must not be nil")
|
||||
}
|
||||
@@ -40,7 +40,7 @@ func (r *Registry) Register(key string, constructor Constructor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) Build(key string) (contracts.InputAdapter, error) {
|
||||
func (r *InputAdapterRegistry) Build(key string) (contracts.InputAdapter, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("input adapter registry must not be nil")
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func (r *Registry) Build(key string) (contracts.InputAdapter, error) {
|
||||
return adapter, nil
|
||||
}
|
||||
|
||||
func (r *Registry) RegisteredKeys() []string {
|
||||
func (r *InputAdapterRegistry) RegisteredKeys() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package inputregistry
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,10 +11,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRegisterAndBuild(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryRegisterAndBuild(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err != nil {
|
||||
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -27,10 +27,10 @@ func TestRegisterAndBuild(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
if err := registry.Register(" generic-input ", fakeConstructor("generic-input")); err != nil {
|
||||
if err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -43,10 +43,10 @@ func TestRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsEmptyKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryRegisterRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
err := registry.Register(" \t", fakeConstructor("generic-input"))
|
||||
err := registry.Register(" \t", fakeInputAdapterConstructor("generic-input"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
@@ -56,13 +56,13 @@ func TestRegisterRejectsEmptyKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
registry := New()
|
||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err != nil {
|
||||
func TestInputAdapterRegistryRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
err := registry.Register(" generic-input ", fakeConstructor("generic-input"))
|
||||
err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
@@ -72,8 +72,8 @@ func TestRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRejectsNilConstructor(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryRegisterRejectsNilConstructor(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
err := registry.Register("generic-input", nil)
|
||||
|
||||
@@ -85,8 +85,8 @@ func TestRegisterRejectsNilConstructor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsUnknownKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryBuildRejectsUnknownKey(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
_, err := registry.Build("missing-input")
|
||||
|
||||
@@ -98,8 +98,8 @@ func TestBuildRejectsUnknownKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWrapsConstructorError(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryBuildWrapsConstructorError(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
constructorErr := errors.New("constructor failed")
|
||||
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
||||
return nil, constructorErr
|
||||
@@ -120,8 +120,8 @@ func TestBuildWrapsConstructorError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsNilAdapter(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryBuildRejectsNilAdapter(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
|
||||
return nil, nil
|
||||
}); err != nil {
|
||||
@@ -138,9 +138,9 @@ func TestBuildRejectsNilAdapter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
||||
registry := New()
|
||||
if err := registry.Register("generic-input", fakeConstructor("other-input")); err != nil {
|
||||
func TestInputAdapterRegistryBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
if err := registry.Register("generic-input", fakeInputAdapterConstructor("other-input")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -154,10 +154,10 @@ func TestBuildRejectsAdapterKeyMismatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
for _, key := range []string{"zeta", "alpha", "middle"} {
|
||||
if err := registry.Register(key, fakeConstructor(key)); err != nil {
|
||||
if err := registry.Register(key, fakeInputAdapterConstructor(key)); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
}
|
||||
@@ -175,10 +175,10 @@ func TestRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilRegistryBehavior(t *testing.T) {
|
||||
var registry *Registry
|
||||
func TestInputAdapterRegistryNilRegistryBehavior(t *testing.T) {
|
||||
var registry *InputAdapterRegistry
|
||||
|
||||
if err := registry.Register("generic-input", fakeConstructor("generic-input")); err == nil {
|
||||
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if _, err := registry.Build("generic-input"); err == nil {
|
||||
@@ -189,8 +189,8 @@ func TestNilRegistryBehavior(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsEmptyKey(t *testing.T) {
|
||||
registry := New()
|
||||
func TestInputAdapterRegistryBuildRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewInputAdapterRegistry()
|
||||
|
||||
_, err := registry.Build(" \n")
|
||||
|
||||
@@ -206,7 +206,7 @@ type fakeAdapter struct {
|
||||
key string
|
||||
}
|
||||
|
||||
func fakeConstructor(key string) Constructor {
|
||||
func fakeInputAdapterConstructor(key string) InputAdapterConstructor {
|
||||
return func() (contracts.InputAdapter, error) {
|
||||
return fakeAdapter{key: key}, nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package runner
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,14 +8,14 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/extractorregistry"
|
||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
|
||||
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||
)
|
||||
|
||||
func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
||||
var builtKeys []string
|
||||
var executedKeys []string
|
||||
registry := extractorregistry.New()
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
registerIntegrationExtractor(t, registry, "second", &builtKeys, &executedKeys, []contracts.Validator{
|
||||
integrationValidator{name: "reject-second", approve: false},
|
||||
@@ -46,7 +46,7 @@ func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func registerIntegrationExtractor(t *testing.T, registry *extractorregistry.Registry, key string, builtKeys *[]string, executedKeys *[]string, validators []contracts.Validator) {
|
||||
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, builtKeys *[]string, executedKeys *[]string, validators []contracts.Validator) {
|
||||
t.Helper()
|
||||
|
||||
if err := registry.Register(key, func() (contracts.Extractor, error) {
|
||||
@@ -101,9 +101,9 @@ func (validator integrationValidator) Validate(ctx context.Context, req contract
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
||||
for _, candidate := range req.Candidates {
|
||||
if validator.approve {
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||
} else {
|
||||
decisions = append(decisions, validationhelpers.Rejected(candidate.Index, "invalid", "not accepted"))
|
||||
decisions = append(decisions, validate.Rejected(candidate.Index, "invalid", "not accepted"))
|
||||
}
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
@@ -1,4 +1,4 @@
|
||||
package runner
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||
)
|
||||
|
||||
type ExtractorFactory interface {
|
||||
@@ -140,7 +140,7 @@ func runValidators(ctx context.Context, extractor contracts.Extractor, doc *sour
|
||||
if result.ValidatorName != validator.Name() {
|
||||
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
|
||||
}
|
||||
if err := validators.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
|
||||
if err := validate.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
|
||||
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractor.Key(), validator.Name(), err)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package runner
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||
)
|
||||
|
||||
func TestNewAndDataTypes(t *testing.T) {
|
||||
@@ -103,7 +103,7 @@ func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
seenIndices = append(seenIndices, candidate.Index)
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||
}
|
||||
return decisions
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func TestRunApprovesCandidatesWithoutValidators(t *testing.T) {
|
||||
|
||||
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||
return []contracts.ValidationDecision{validate.Approved(candidates[0].Index)}
|
||||
}}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{validator}},
|
||||
@@ -214,13 +214,13 @@ func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T)
|
||||
var laterSeen int
|
||||
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||
return []contracts.ValidationDecision{
|
||||
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
||||
validationhelpers.Approved(candidates[1].Index),
|
||||
validate.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
||||
validate.Approved(candidates[1].Index),
|
||||
}
|
||||
}}
|
||||
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||
laterSeen = len(candidates)
|
||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||
return []contracts.ValidationDecision{validate.Approved(candidates[0].Index)}
|
||||
}}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{rejectFirst, approveRemaining}},
|
||||
@@ -439,7 +439,7 @@ func factoryWithValidator(validator contracts.Validator) fakeFactory {
|
||||
func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||
}
|
||||
return decisions
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package validators
|
||||
package validate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package validators
|
||||
package validate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
Reference in New Issue
Block a user