Update the architecture plan to standardize on input -> chunk -> extract -> merge -> normalize -> output naming conventions
This commit is contained in:
@@ -29,7 +29,7 @@ Source-format details belong in input modules. Transcript-specific concepts
|
|||||||
such as segments, speakers, timestamps, and transcript schemas must not spread
|
such as segments, speakers, timestamps, and transcript schemas must not spread
|
||||||
into runner, extractor, or validator framework code.
|
into runner, extractor, or validator framework code.
|
||||||
|
|
||||||
Extraction-domain details belong in process modules. D&D-specific concepts
|
Extraction-domain details belong in extract modules. D&D-specific concepts
|
||||||
such as spells, NPCs, items, combat turns, and encounters must not spread into
|
such as spells, NPCs, items, combat turns, and encounters must not spread into
|
||||||
core source, runner, or LLM framework packages.
|
core source, runner, or LLM framework packages.
|
||||||
|
|
||||||
@@ -39,11 +39,11 @@ should point to generic source units, not to transcript-only structures.
|
|||||||
The application workflow is:
|
The application workflow is:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
These stages should remain explicit in the architecture. Chunking, merging, and
|
These stages should remain explicit in the architecture. Chunking, merging, and
|
||||||
normalization must not be hidden inside domain process modules when they represent
|
normalization must not be hidden inside domain extract modules when they represent
|
||||||
general pipeline behavior.
|
general pipeline behavior.
|
||||||
|
|
||||||
## Dependency Policy
|
## Dependency Policy
|
||||||
@@ -99,7 +99,7 @@ Domain implementations:
|
|||||||
|
|
||||||
- `internal/modules/input/<name>`: input-stage modules that parse external input into core source documents.
|
- `internal/modules/input/<name>`: input-stage modules that parse external input into core source documents.
|
||||||
- `internal/modules/chunk/<name>`: chunk-stage modules.
|
- `internal/modules/chunk/<name>`: chunk-stage modules.
|
||||||
- `internal/modules/process/<domain>/<name>`: process-stage extractor modules.
|
- `internal/modules/extract/<domain>/<name>`: extract-stage extractor modules.
|
||||||
- `internal/modules/merge/<name>` or `internal/modules/merge/<domain>/<name>`: merge-stage modules.
|
- `internal/modules/merge/<name>` or `internal/modules/merge/<domain>/<name>`: merge-stage modules.
|
||||||
- `internal/modules/normalize/<name>` or `internal/modules/normalize/<domain>/<name>`: normalize-stage modules.
|
- `internal/modules/normalize/<name>` or `internal/modules/normalize/<domain>/<name>`: normalize-stage modules.
|
||||||
- `internal/modules/output/<name>`: output-stage modules.
|
- `internal/modules/output/<name>`: output-stage modules.
|
||||||
@@ -120,7 +120,7 @@ workflow visible in the filesystem:
|
|||||||
```text
|
```text
|
||||||
internal/modules/input/...
|
internal/modules/input/...
|
||||||
internal/modules/chunk/...
|
internal/modules/chunk/...
|
||||||
internal/modules/process/...
|
internal/modules/extract/...
|
||||||
internal/modules/merge/...
|
internal/modules/merge/...
|
||||||
internal/modules/normalize/...
|
internal/modules/normalize/...
|
||||||
internal/modules/output/...
|
internal/modules/output/...
|
||||||
@@ -141,7 +141,7 @@ decisions.
|
|||||||
|
|
||||||
Other packages should interact with source input through adapter contracts and
|
Other packages should interact with source input through adapter contracts and
|
||||||
core source types. Input module implementation details and external dependency
|
core source types. Input module implementation details and external dependency
|
||||||
types must not leak into framework or process module packages.
|
types must not leak into framework or extract module packages.
|
||||||
|
|
||||||
Input module metadata may preserve source-specific facts such as transcript speaker,
|
Input module metadata may preserve source-specific facts such as transcript speaker,
|
||||||
timestamps, Markdown heading path, page number, or block ID. Framework code may
|
timestamps, Markdown heading path, page number, or block ID. Framework code may
|
||||||
@@ -152,7 +152,7 @@ shape.
|
|||||||
|
|
||||||
Extractors are independent modules that process source chunks or whole source
|
Extractors are independent modules that process source chunks or whole source
|
||||||
documents and produce one kind of structured artifact candidate.
|
documents and produce one kind of structured artifact candidate.
|
||||||
Each process module owns:
|
Each extract module owns:
|
||||||
|
|
||||||
- its artifact semantics;
|
- its artifact semantics;
|
||||||
- its prompt usage;
|
- its prompt usage;
|
||||||
@@ -160,7 +160,7 @@ Each process module owns:
|
|||||||
- its validator chain;
|
- its validator chain;
|
||||||
- any domain-specific mapping or interpretation.
|
- any domain-specific mapping or interpretation.
|
||||||
|
|
||||||
Process modules should depend on framework contracts and core source/artifact
|
Extract modules should depend on framework contracts and core source/artifact
|
||||||
types. They should not depend on concrete input module packages.
|
types. They should not depend on concrete input module packages.
|
||||||
|
|
||||||
Extractors should not be the only place where chunking, merging, or
|
Extractors should not be the only place where chunking, merging, or
|
||||||
@@ -181,7 +181,7 @@ The pipeline has six conceptual stages:
|
|||||||
|
|
||||||
1. input: external source material becomes a `SourceDocument`;
|
1. input: external source material becomes a `SourceDocument`;
|
||||||
2. chunk: a `SourceDocument` becomes ordered source chunks;
|
2. chunk: a `SourceDocument` becomes ordered source chunks;
|
||||||
3. process: extractors produce artifact candidates from chunks or whole documents;
|
3. extract: extractors produce artifact candidates from chunks or whole documents;
|
||||||
4. merge: per-chunk candidates become a merged candidate collection;
|
4. merge: per-chunk candidates become a merged candidate collection;
|
||||||
5. normalize: merged candidates are reconciled for duplicates, aliases, consistency, or cross-chunk issues;
|
5. normalize: merged candidates are reconciled for duplicates, aliases, consistency, or cross-chunk issues;
|
||||||
6. output: final artifacts are serialized.
|
6. output: final artifacts are serialized.
|
||||||
@@ -290,11 +290,11 @@ be opt-in.
|
|||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Core logic should be testable without real external services. Use fakes,
|
Core logic should be testable without real external services. Use fakes,
|
||||||
fixtures, or local test doubles for adapters, extractors, validators, and LLM
|
fixtures, or local test doubles for input modules, extract modules, validators,
|
||||||
clients where practical.
|
and LLM clients where practical.
|
||||||
|
|
||||||
Contract-first work should include fake implementations that prove interfaces
|
Contract-first work should include fake implementations that prove interfaces
|
||||||
compose before real adapters or extractors depend on them.
|
compose before real input modules or extract modules depend on them.
|
||||||
|
|
||||||
Config examples should be load-tested once config files exist. Important CLI
|
Config examples should be load-tested once config files exist. Important CLI
|
||||||
workflows should have parser or command tests. Adapter, extractor, and validator
|
workflows should have parser or command tests. Adapter, extractor, and validator
|
||||||
@@ -312,7 +312,7 @@ unit, source reference, input adapter, extractor, chunker, merger, normalizer,
|
|||||||
artifact, validator, and run manifest.
|
artifact, validator, and run manifest.
|
||||||
|
|
||||||
Source-format details belong in input module or integration docs.
|
Source-format details belong in input module or integration docs.
|
||||||
Domain-specific extraction details belong in process module or artifact docs.
|
Domain-specific extraction details belong in extract module or artifact docs.
|
||||||
|
|
||||||
When changing architecture, config, CLI behavior, stage modules, extractor
|
When changing architecture, config, CLI behavior, stage modules, extractor
|
||||||
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
|
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ In scope:
|
|||||||
Out of scope:
|
Out of scope:
|
||||||
|
|
||||||
- real input modules;
|
- real input modules;
|
||||||
- real process modules;
|
- real extract modules;
|
||||||
- LLM provider calls;
|
- LLM provider calls;
|
||||||
- prompt or response-schema assets;
|
- prompt or response-schema assets;
|
||||||
- diagnostics run directory;
|
- diagnostics run directory;
|
||||||
@@ -47,7 +47,7 @@ contract packages:
|
|||||||
structured LLM interfaces used by later checkpoints.
|
structured LLM interfaces used by later checkpoints.
|
||||||
|
|
||||||
The contracts should be proven with fake implementations in tests. Those tests
|
The contracts should be proven with fake implementations in tests. Those tests
|
||||||
should demonstrate composition without real input modules, real process modules,
|
should demonstrate composition without real input modules, real extract modules,
|
||||||
LLM provider calls, prompt assets, or diagnostics infrastructure.
|
LLM provider calls, prompt assets, or diagnostics infrastructure.
|
||||||
|
|
||||||
Implementation staging belongs in
|
Implementation staging belongs in
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ Out of scope:
|
|||||||
|
|
||||||
- real input parsing;
|
- real input parsing;
|
||||||
- real input modules;
|
- real input modules;
|
||||||
- real process modules;
|
- real extract modules;
|
||||||
- real LLM calls;
|
- real LLM calls;
|
||||||
- prompt assets;
|
- prompt assets;
|
||||||
- response schema assets;
|
- response schema assets;
|
||||||
@@ -60,7 +60,7 @@ Implementation staging belongs in
|
|||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Fake adapter/extractor/validator registrations work in tests.
|
- Fake adapter/extractor/validator registrations work in tests.
|
||||||
- The runner operates on `SourceDocument`, not transcript-specific structures.
|
- The runner operates on `SourceDocument`, not transcript-specific structures.
|
||||||
- The runner does not import concrete D&D process module packages.
|
- The runner does not import concrete D&D extract module packages.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
|
|||||||
@@ -7,17 +7,17 @@ This document describes planned work, not implemented behavior.
|
|||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Make Notarius's application workflow first-class before adding real input
|
Make Notarius's application workflow first-class before adding real input
|
||||||
modules or process modules.
|
modules or extract modules.
|
||||||
|
|
||||||
The workflow should be:
|
The workflow should be:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
Checkpoint 3 should define the contracts and minimal fake-tested framework
|
Checkpoint 3 should define the contracts and minimal fake-tested framework
|
||||||
behavior for chunking, per-chunk processing, merging, and normalization. It
|
behavior for chunking, per-chunk processing, merging, and normalization. It
|
||||||
should not add real input modules, real domain process modules, LLM provider code,
|
should not add real input modules, real domain extract modules, LLM provider code,
|
||||||
or output encoders.
|
or output encoders.
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
@@ -26,7 +26,7 @@ In scope:
|
|||||||
|
|
||||||
- source chunk model;
|
- source chunk model;
|
||||||
- chunker contract;
|
- chunker contract;
|
||||||
- process-stage contract for extractors operating on chunks;
|
- extract-stage contract for extractors operating on chunks;
|
||||||
- merge-stage contract;
|
- merge-stage contract;
|
||||||
- normalize-stage contract;
|
- normalize-stage contract;
|
||||||
- output-stage contract shape, if useful for pipeline completeness;
|
- output-stage contract shape, if useful for pipeline completeness;
|
||||||
@@ -50,13 +50,13 @@ The repository should contain explicit pipeline-stage contracts:
|
|||||||
|
|
||||||
- `InputAdapter`: external source input to `SourceDocument`.
|
- `InputAdapter`: external source input to `SourceDocument`.
|
||||||
- `Chunker`: `SourceDocument` to ordered `SourceChunk` values.
|
- `Chunker`: `SourceDocument` to ordered `SourceChunk` values.
|
||||||
- `Extractor` or processor: `SourceChunk` to artifact candidates.
|
- `Extractor`: `SourceChunk` to artifact candidates.
|
||||||
- `Merger`: per-chunk candidates to merged candidates.
|
- `Merger`: per-chunk candidates to merged candidates.
|
||||||
- `Normalizer`: merged candidates to normalized candidates.
|
- `Normalizer`: merged candidates to normalized candidates.
|
||||||
- `OutputEncoder`: final artifact bundle to bytes, if introduced in this
|
- `OutputEncoder`: final artifact bundle to bytes, if introduced in this
|
||||||
checkpoint.
|
checkpoint.
|
||||||
|
|
||||||
The runner should orchestrate fake implementations through chunk, process,
|
The runner should orchestrate fake implementations through chunk, extract,
|
||||||
merge, normalize, and approval/validation behavior in tests.
|
merge, normalize, and approval/validation behavior in tests.
|
||||||
|
|
||||||
## Design Intent
|
## Design Intent
|
||||||
@@ -113,17 +113,17 @@ Domain-specific normalizers may later:
|
|||||||
|
|
||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Pipeline-stage contracts are explicit and source/domain agnostic.
|
- Pipeline-stage contracts are explicit and source/domain agnostic.
|
||||||
- Fake tests prove input source documents can be chunked, processed, merged, and
|
- Fake tests prove input source documents can be chunked, extracted, merged, and
|
||||||
normalized.
|
normalized.
|
||||||
- Merge and normalize are distinct concepts in code and tests.
|
- Merge and normalize are distinct concepts in code and tests.
|
||||||
- The runner no longer implies whole-document-only extraction as the core
|
- The runner no longer implies whole-document-only extraction as the core
|
||||||
application model.
|
application model.
|
||||||
- No concrete input module, process module, LLM provider, prompt, response schema,
|
- No concrete input module, extract module, LLM provider, prompt, response schema,
|
||||||
diagnostics, config, or D&D artifact code is added.
|
diagnostics, config, or D&D artifact code is added.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
- Is the workflow clearly represented as input, chunk, process, merge,
|
- Is the workflow clearly represented as input, chunk, extract, merge,
|
||||||
normalize, and output?
|
normalize, and output?
|
||||||
- Are merge and normalize cleanly separated?
|
- Are merge and normalize cleanly separated?
|
||||||
- Can a generic merger handle simple chronological artifact streams?
|
- Can a generic merger handle simple chronological artifact streams?
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ This document describes planned work, not implemented behavior.
|
|||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Implement the first useful process-stage module: D&D spell casts from a
|
Implement the first useful extract-stage module: D&D spell casts from a
|
||||||
Seriatim transcript source document.
|
Seriatim transcript source document.
|
||||||
|
|
||||||
This checkpoint should produce the first meaningful vertical slice from real
|
This checkpoint should produce the first meaningful vertical slice from real
|
||||||
@@ -19,7 +19,7 @@ In scope:
|
|||||||
- D&D spell artifact schema and Go structs;
|
- D&D spell artifact schema and Go structs;
|
||||||
- structured response schema asset;
|
- structured response schema asset;
|
||||||
- prompt assets;
|
- prompt assets;
|
||||||
- `internal/modules/process/dnd/spells`;
|
- `internal/modules/extract/dnd/spells`;
|
||||||
- source-reference and schema validators in the extractor chain;
|
- source-reference and schema validators in the extractor chain;
|
||||||
- fake LLM tests;
|
- fake LLM tests;
|
||||||
- CLI-level integration test if the CLI path is ready.
|
- CLI-level integration test if the CLI path is ready.
|
||||||
@@ -50,7 +50,7 @@ type SpellCast struct {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Keep this schema inside the D&D spells process module or a D&D artifact package,
|
Keep this schema inside the D&D spells extract module or a D&D artifact package,
|
||||||
not inside core framework packages.
|
not inside core framework packages.
|
||||||
|
|
||||||
### Stage 2: Structured Response Schema
|
### Stage 2: Structured Response Schema
|
||||||
@@ -76,7 +76,7 @@ Prompts should:
|
|||||||
|
|
||||||
### Stage 4: Process Module Implementation
|
### Stage 4: Process Module Implementation
|
||||||
|
|
||||||
Implement `internal/modules/process/dnd/spells`.
|
Implement `internal/modules/extract/dnd/spells`.
|
||||||
|
|
||||||
The extractor should:
|
The extractor should:
|
||||||
|
|
||||||
@@ -117,13 +117,13 @@ The test should use fake LLM wiring and fixture input.
|
|||||||
- `go test ./...` passes.
|
- `go test ./...` passes.
|
||||||
- Seriatim input can flow through the runner into the D&D spells extractor.
|
- Seriatim input can flow through the runner into the D&D spells extractor.
|
||||||
- Spell artifacts include valid source references.
|
- Spell artifacts include valid source references.
|
||||||
- D&D concepts are contained in process module/artifact packages and docs.
|
- D&D concepts are contained in extract module/artifact packages and docs.
|
||||||
- The first meaningful vertical slice is available through tests, and through
|
- The first meaningful vertical slice is available through tests, and through
|
||||||
CLI if the CLI path is ready.
|
CLI if the CLI path is ready.
|
||||||
|
|
||||||
## Review Questions
|
## Review Questions
|
||||||
|
|
||||||
- Is the spell process module domain-specific without making the framework
|
- Is the spell extract module domain-specific without making the framework
|
||||||
D&D-specific?
|
D&D-specific?
|
||||||
- Are source references valid and useful for downstream validation?
|
- Are source references valid and useful for downstream validation?
|
||||||
- Is prompt/schema ownership clear?
|
- Is prompt/schema ownership clear?
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ not describe implemented behavior.
|
|||||||
Notarius documentation should make three boundaries obvious:
|
Notarius documentation should make three boundaries obvious:
|
||||||
|
|
||||||
- source-format support belongs to input-stage modules;
|
- source-format support belongs to input-stage modules;
|
||||||
- extraction-domain behavior belongs to process-stage modules;
|
- extraction-domain behavior belongs to extract-stage modules;
|
||||||
- core framework behavior is source-agnostic and domain-agnostic.
|
- core framework behavior is source-agnostic and domain-agnostic.
|
||||||
|
|
||||||
Documentation should avoid making the MVP look more transcript-specific or
|
Documentation should avoid making the MVP look more transcript-specific or
|
||||||
@@ -47,7 +47,7 @@ Core docs should avoid transcript-specific terms such as segment, speaker,
|
|||||||
timestamp, and transcript range unless discussing an input adapter or an example.
|
timestamp, and transcript range unless discussing an input adapter or an example.
|
||||||
|
|
||||||
Core docs should avoid D&D-specific terms such as spell, NPC, item, combat, and
|
Core docs should avoid D&D-specific terms such as spell, NPC, item, combat, and
|
||||||
encounter unless discussing process modules, artifact docs, or examples.
|
encounter unless discussing extract modules, artifact docs, or examples.
|
||||||
|
|
||||||
### Input Module Docs Own Source Formats
|
### Input Module Docs Own Source Formats
|
||||||
|
|
||||||
@@ -95,12 +95,12 @@ Stage module docs should cover:
|
|||||||
- prompt and response-schema ownership;
|
- prompt and response-schema ownership;
|
||||||
- examples.
|
- examples.
|
||||||
|
|
||||||
D&D concepts should be documented in D&D process-module or artifact docs, not in
|
D&D concepts should be documented in D&D extract-module or artifact docs, not in
|
||||||
generic runner or framework docs.
|
generic runner or framework docs.
|
||||||
|
|
||||||
### CLI Docs Should Reflect Extensibility
|
### CLI Docs Should Reflect Extensibility
|
||||||
|
|
||||||
The CLI reference should present input modules, chunk modules, process modules,
|
The CLI reference should present input modules, chunk modules, extract modules,
|
||||||
merge modules, normalize modules, and output modules as selectable or
|
merge modules, normalize modules, and output modules as selectable or
|
||||||
configurable components as they become user-facing.
|
configurable components as they become user-facing.
|
||||||
|
|
||||||
@@ -114,7 +114,7 @@ Once implemented, `docs/cli.md` should document:
|
|||||||
|
|
||||||
- positional source input path;
|
- positional source input path;
|
||||||
- input module selection;
|
- input module selection;
|
||||||
- process module selection;
|
- extract module selection;
|
||||||
- chunk/merge/normalize/output selection when configurable;
|
- chunk/merge/normalize/output selection when configurable;
|
||||||
- config path behavior;
|
- config path behavior;
|
||||||
- output path behavior;
|
- output path behavior;
|
||||||
@@ -127,7 +127,7 @@ Once implemented, `docs/cli.md` should document:
|
|||||||
|
|
||||||
- input module selection and module-specific options;
|
- input module selection and module-specific options;
|
||||||
- chunk module selection and module-specific options;
|
- chunk module selection and module-specific options;
|
||||||
- process module selection and module-specific options;
|
- extract module selection and module-specific options;
|
||||||
- merge module selection and module-specific options;
|
- merge module selection and module-specific options;
|
||||||
- normalize module selection and module-specific options;
|
- normalize module selection and module-specific options;
|
||||||
- output module selection and module-specific options;
|
- output module selection and module-specific options;
|
||||||
@@ -176,7 +176,7 @@ Before merging docs, check:
|
|||||||
|
|
||||||
- Does the document describe implemented behavior outside `docs/roadmap/`?
|
- Does the document describe implemented behavior outside `docs/roadmap/`?
|
||||||
- Are source-format details isolated to input module or integration docs?
|
- Are source-format details isolated to input module or integration docs?
|
||||||
- Are D&D details isolated to process module or artifact docs?
|
- Are D&D details isolated to extract module or artifact docs?
|
||||||
- Is there one canonical home for the topic?
|
- Is there one canonical home for the topic?
|
||||||
- Do command examples match implemented CLI syntax?
|
- Do command examples match implemented CLI syntax?
|
||||||
- Are examples valid, maintained, and free of secrets?
|
- Are examples valid, maintained, and free of secrets?
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ This is a staged implementation plan for
|
|||||||
an LLM coding agent to follow stage by stage.
|
an LLM coding agent to follow stage by stage.
|
||||||
|
|
||||||
This plan implements only checkpoint 2. Do not implement real input modules,
|
This plan implements only checkpoint 2. Do not implement real input modules,
|
||||||
real process modules, source chunking, LLM provider clients, prompt assets,
|
real extract modules, source chunking, LLM provider clients, prompt assets,
|
||||||
response schema registries, diagnostics run directories, production config
|
response schema registries, diagnostics run directories, production config
|
||||||
loading, or D&D artifact schemas in this pass.
|
loading, or D&D artifact schemas in this pass.
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ Required boundaries:
|
|||||||
|
|
||||||
- framework packages must stay source-agnostic and domain-agnostic;
|
- framework packages must stay source-agnostic and domain-agnostic;
|
||||||
- input adapter registry code must not import concrete input module packages;
|
- input adapter registry code must not import concrete input module packages;
|
||||||
- extractor registry and runner code must not import concrete process module
|
- extractor registry and runner code must not import concrete extract module
|
||||||
packages;
|
packages;
|
||||||
- runner code should operate on `SourceDocument`, not transcript-specific
|
- runner code should operate on `SourceDocument`, not transcript-specific
|
||||||
structures;
|
structures;
|
||||||
@@ -196,7 +196,7 @@ Cover the same cases as the input adapter registry:
|
|||||||
- sorted `RegisteredKeys`;
|
- sorted `RegisteredKeys`;
|
||||||
- nil registry behavior.
|
- nil registry behavior.
|
||||||
|
|
||||||
Use fake extractors only. Do not add concrete process module packages.
|
Use fake extractors only. Do not add concrete extract module packages.
|
||||||
|
|
||||||
### Validation
|
### Validation
|
||||||
|
|
||||||
@@ -236,7 +236,7 @@ Define helpers:
|
|||||||
```go
|
```go
|
||||||
func Approved(candidateIndex int) contracts.ValidationDecision
|
func Approved(candidateIndex int) contracts.ValidationDecision
|
||||||
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision
|
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision
|
||||||
func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []contracts.ValidationDecision) error
|
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error
|
||||||
```
|
```
|
||||||
|
|
||||||
### Required Behavior
|
### Required Behavior
|
||||||
@@ -264,8 +264,8 @@ func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []co
|
|||||||
- work with candidate indices, not slice positions;
|
- work with candidate indices, not slice positions;
|
||||||
- allow an empty candidate slice only when decisions are also empty.
|
- allow an empty candidate slice only when decisions are also empty.
|
||||||
|
|
||||||
Candidate indices are expected to be unique by the time validators run. If the
|
Artifact candidate indices are expected to be unique by the time validators run.
|
||||||
candidate slice itself contains duplicate indices, return an error.
|
If the candidate slice itself contains duplicate indices, return an error.
|
||||||
|
|
||||||
### Required Tests
|
### Required Tests
|
||||||
|
|
||||||
@@ -396,7 +396,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error)
|
|||||||
1. reject nil runner or nil extractor factory;
|
1. reject nil runner or nil extractor factory;
|
||||||
2. validate `input.Source` with `source.ValidateDocument`;
|
2. validate `input.Source` with `source.ValidateDocument`;
|
||||||
3. reject an empty `ExtractorKeys` list;
|
3. reject an empty `ExtractorKeys` list;
|
||||||
4. process extractor keys in input order;
|
4. iterate extractor keys in input order;
|
||||||
5. build each extractor through the factory;
|
5. build each extractor through the factory;
|
||||||
6. call `Extractor.Extract` with the source, LLM client, and metadata;
|
6. call `Extractor.Extract` with the source, LLM client, and metadata;
|
||||||
7. append extractor warnings to `RunOutput.Warnings`;
|
7. append extractor warnings to `RunOutput.Warnings`;
|
||||||
@@ -406,9 +406,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error)
|
|||||||
11. reject candidates denied by validators;
|
11. reject candidates denied by validators;
|
||||||
12. return approved/rejected artifacts in deterministic order.
|
12. return approved/rejected artifacts in deterministic order.
|
||||||
|
|
||||||
Candidate normalization must:
|
Artifact candidate normalization must:
|
||||||
|
|
||||||
- assign `Candidate.Index` using a global monotonically increasing counter;
|
- assign `ArtifactCandidate.Index` using a global monotonically increasing counter;
|
||||||
- fill empty `ExtractorKey`, `ArtifactType`, and `SchemaVersion` from the
|
- fill empty `ExtractorKey`, `ArtifactType`, and `SchemaVersion` from the
|
||||||
owning extractor;
|
owning extractor;
|
||||||
- return an error if a candidate's non-empty `ExtractorKey` differs from
|
- return an error if a candidate's non-empty `ExtractorKey` differs from
|
||||||
@@ -502,7 +502,7 @@ Assertions:
|
|||||||
- extractor execution follows configured key order;
|
- extractor execution follows configured key order;
|
||||||
- approved artifacts are in deterministic order;
|
- approved artifacts are in deterministic order;
|
||||||
- rejected artifacts are in deterministic order;
|
- rejected artifacts are in deterministic order;
|
||||||
- no concrete input module or process module packages are imported.
|
- no concrete input module or extract module packages are imported.
|
||||||
|
|
||||||
### Validation
|
### Validation
|
||||||
|
|
||||||
@@ -528,7 +528,7 @@ Check:
|
|||||||
|
|
||||||
- no package names mention transcripts, Seriatim, D&D, spells, NPCs, items, or
|
- no package names mention transcripts, Seriatim, D&D, spells, NPCs, items, or
|
||||||
combat;
|
combat;
|
||||||
- no concrete input module or process module package exists;
|
- no concrete input module or extract module package exists;
|
||||||
- no LLM provider code exists;
|
- no LLM provider code exists;
|
||||||
- no prompt, response schema, diagnostics, config, or source chunking package
|
- no prompt, response schema, diagnostics, config, or source chunking package
|
||||||
was added;
|
was added;
|
||||||
|
|||||||
@@ -14,10 +14,10 @@ The first MVP should target audio transcripts generated by Seriatim. That
|
|||||||
choice should be implemented as an input-stage module, not as a
|
choice should be implemented as an input-stage module, not as a
|
||||||
transcript-specific assumption in the application core. Later input sources,
|
transcript-specific assumption in the application core. Later input sources,
|
||||||
such as unstructured Markdown notes or Obsidian documents, should be addable
|
such as unstructured Markdown notes or Obsidian documents, should be addable
|
||||||
through new input and process modules without reshaping the framework.
|
through new input and extract modules without reshaping the framework.
|
||||||
|
|
||||||
The first extraction domain should be D&D session analysis, starting with spell
|
The first extraction domain should be D&D session analysis, starting with spell
|
||||||
casts. That domain should live in process-stage modules and related schemas, not
|
casts. That domain should live in extract-stage modules and related schemas, not
|
||||||
in core framework packages.
|
in core framework packages.
|
||||||
|
|
||||||
The application should follow the same broad architecture as Audita:
|
The application should follow the same broad architecture as Audita:
|
||||||
@@ -25,7 +25,7 @@ The application should follow the same broad architecture as Audita:
|
|||||||
- deterministic core packages for config, source documents, artifacts, diagnostics, and reporting;
|
- deterministic core packages for config, source documents, artifacts, diagnostics, and reporting;
|
||||||
- input-stage modules that translate external source formats into a small internal source model;
|
- input-stage modules that translate external source formats into a small internal source model;
|
||||||
- reusable framework packages for contracts, orchestration, LLM runtime, structured output, and validation;
|
- reusable framework packages for contracts, orchestration, LLM runtime, structured output, and validation;
|
||||||
- independent process-stage modules that own domain-specific behavior;
|
- independent extract-stage modules that own domain-specific behavior;
|
||||||
- independent validator packages;
|
- independent validator packages;
|
||||||
- embedded prompt and JSON schema assets;
|
- embedded prompt and JSON schema assets;
|
||||||
- CLI orchestration that wires the pieces together without owning domain logic.
|
- CLI orchestration that wires the pieces together without owning domain logic.
|
||||||
@@ -37,7 +37,7 @@ artifacts rather than proposing and applying transcript corrections.
|
|||||||
|
|
||||||
- Keep the core input model generic: ordered text units plus metadata.
|
- Keep the core input model generic: ordered text units plus metadata.
|
||||||
- Keep source-format details in hexagonal input modules.
|
- Keep source-format details in hexagonal input modules.
|
||||||
- Keep extraction-domain details in process modules.
|
- Keep extraction-domain details in extract modules.
|
||||||
- Treat evidence as source references, not transcript references.
|
- Treat evidence as source references, not transcript references.
|
||||||
- Prefer narrow, useful abstractions over a universal document model.
|
- Prefer narrow, useful abstractions over a universal document model.
|
||||||
- Preserve enough provenance for validation, replay, and downstream inspection.
|
- Preserve enough provenance for validation, replay, and downstream inspection.
|
||||||
@@ -77,10 +77,10 @@ internal/modules/input/markdown
|
|||||||
internal/modules/chunk/generic
|
internal/modules/chunk/generic
|
||||||
internal/modules/chunk/dndtranscript
|
internal/modules/chunk/dndtranscript
|
||||||
|
|
||||||
internal/modules/process/dnd/spells
|
internal/modules/extract/dnd/spells
|
||||||
internal/modules/process/dnd/items
|
internal/modules/extract/dnd/items
|
||||||
internal/modules/process/dnd/npcs
|
internal/modules/extract/dnd/npcs
|
||||||
internal/modules/process/dnd/combat
|
internal/modules/extract/dnd/combat
|
||||||
|
|
||||||
internal/modules/merge/appendorder
|
internal/modules/merge/appendorder
|
||||||
internal/modules/merge/dnd/spells
|
internal/modules/merge/dnd/spells
|
||||||
@@ -193,7 +193,7 @@ type Extractor interface {
|
|||||||
ArtifactType() string
|
ArtifactType() string
|
||||||
SchemaVersion() string
|
SchemaVersion() string
|
||||||
Validators() []Validator
|
Validators() []Validator
|
||||||
Process(ctx context.Context, req ProcessRequest) (ProcessResult, error)
|
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -201,9 +201,9 @@ An extractor should receive either a whole source document or a source chunk,
|
|||||||
depending on processing mode. It should return typed artifact candidates plus
|
depending on processing mode. It should return typed artifact candidates plus
|
||||||
warnings. It should not mutate the source document.
|
warnings. It should not mutate the source document.
|
||||||
|
|
||||||
Process modules own domain concepts. For example, D&D spell extraction should
|
Extract modules own domain concepts. For example, D&D spell extraction should
|
||||||
live under `internal/modules/process/dnd/spells`; a future to-do extractor for
|
live under `internal/modules/extract/dnd/spells`; a future to-do extractor for
|
||||||
notes should live under a different process-module path and use the same
|
notes should live under a different extract-module path and use the same
|
||||||
framework contract.
|
framework contract.
|
||||||
|
|
||||||
### Chunker
|
### Chunker
|
||||||
@@ -262,9 +262,13 @@ Per-run provenance record.
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
InputAdapter string `json:"input_adapter"`
|
InputModule string `json:"input_module"`
|
||||||
|
Chunker string `json:"chunker"`
|
||||||
SourceDigests []string `json:"source_digests"`
|
SourceDigests []string `json:"source_digests"`
|
||||||
Extractors []string `json:"extractors"`
|
Extractors []string `json:"extractors"`
|
||||||
|
Merger string `json:"merger"`
|
||||||
|
Normalizer string `json:"normalizer"`
|
||||||
|
OutputEncoder string `json:"output_encoder"`
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
ValidationStatus string `json:"validation_status"`
|
ValidationStatus string `json:"validation_status"`
|
||||||
}
|
}
|
||||||
@@ -345,7 +349,7 @@ The architecture should support extractors outside the D&D domain. Examples:
|
|||||||
- decisions and action items from meeting transcripts;
|
- decisions and action items from meeting transcripts;
|
||||||
- named people, places, and dates from research notes.
|
- named people, places, and dates from research notes.
|
||||||
|
|
||||||
These should be addable as process modules without changing runner,
|
These should be addable as extract modules without changing runner,
|
||||||
validator, source-reference, or LLM framework contracts.
|
validator, source-reference, or LLM framework contracts.
|
||||||
|
|
||||||
## Proposed Pipeline Flow
|
## Proposed Pipeline Flow
|
||||||
@@ -353,7 +357,7 @@ validator, source-reference, or LLM framework contracts.
|
|||||||
The application workflow should be first-class:
|
The application workflow should be first-class:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
input -> chunk -> process -> merge -> normalize -> output
|
input -> chunk -> extract -> merge -> normalize -> output
|
||||||
```
|
```
|
||||||
|
|
||||||
Proposed runner flow:
|
Proposed runner flow:
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Candidate struct {
|
type ArtifactCandidate struct {
|
||||||
Index int `json:"index"`
|
Index int `json:"index"`
|
||||||
ExtractorKey string `json:"extractor_key"`
|
ExtractorKey string `json:"extractor_key"`
|
||||||
ArtifactType string `json:"artifact_type"`
|
ArtifactType string `json:"artifact_type"`
|
||||||
@@ -27,7 +27,7 @@ type Artifact struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type RejectedArtifact struct {
|
type RejectedArtifact struct {
|
||||||
Candidate Candidate `json:"candidate"`
|
Candidate ArtifactCandidate `json:"candidate"`
|
||||||
ValidatorName string `json:"validator_name"`
|
ValidatorName string `json:"validator_name"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
@@ -35,16 +35,20 @@ type RejectedArtifact struct {
|
|||||||
|
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
InputAdapter string `json:"input_adapter,omitempty"`
|
InputModule string `json:"input_module,omitempty"`
|
||||||
|
Chunker string `json:"chunker,omitempty"`
|
||||||
SourceDigests []string `json:"source_digests,omitempty"`
|
SourceDigests []string `json:"source_digests,omitempty"`
|
||||||
Extractors []string `json:"extractors,omitempty"`
|
Extractors []string `json:"extractors,omitempty"`
|
||||||
|
Merger string `json:"merger,omitempty"`
|
||||||
|
Normalizer string `json:"normalizer,omitempty"`
|
||||||
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
SchemaVersion string `json:"schema_version,omitempty"`
|
SchemaVersion string `json:"schema_version,omitempty"`
|
||||||
ValidationStatus string `json:"validation_status,omitempty"`
|
ValidationStatus string `json:"validation_status,omitempty"`
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ArtifactFromCandidate(candidate Candidate) Artifact {
|
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
||||||
return Artifact{
|
return Artifact{
|
||||||
ExtractorKey: candidate.ExtractorKey,
|
ExtractorKey: candidate.ExtractorKey,
|
||||||
ArtifactType: candidate.ArtifactType,
|
ArtifactType: candidate.ArtifactType,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
||||||
candidate := Candidate{
|
candidate := ArtifactCandidate{
|
||||||
Index: 7,
|
Index: 7,
|
||||||
ExtractorKey: "generic-extractor",
|
ExtractorKey: "generic-extractor",
|
||||||
ArtifactType: "generic-artifact",
|
ArtifactType: "generic-artifact",
|
||||||
@@ -60,7 +60,7 @@ func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
||||||
candidate := Candidate{
|
candidate := ArtifactCandidate{
|
||||||
Index: 1,
|
Index: 1,
|
||||||
ExtractorKey: "generic-extractor",
|
ExtractorKey: "generic-extractor",
|
||||||
ArtifactType: "generic-artifact",
|
ArtifactType: "generic-artifact",
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ func (extractor compositionExtractor) Extract(ctx context.Context, req contracts
|
|||||||
}
|
}
|
||||||
|
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{
|
{
|
||||||
Index: 0,
|
Index: 0,
|
||||||
ExtractorKey: extractor.Key(),
|
ExtractorKey: extractor.Key(),
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ type ExtractionRequest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ExtractionResult struct {
|
type ExtractionResult struct {
|
||||||
Candidates []artifacts.Candidate `json:"candidates,omitempty"`
|
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,7 +67,7 @@ type Extractor interface {
|
|||||||
|
|
||||||
type ValidationRequest struct {
|
type ValidationRequest struct {
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
Candidates []artifacts.Candidate `json:"candidates"`
|
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -58,19 +58,19 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
|
|||||||
|
|
||||||
candidate := result.Candidates[0]
|
candidate := result.Candidates[0]
|
||||||
if candidate.Index != 0 {
|
if candidate.Index != 0 {
|
||||||
t.Fatalf("Candidate.Index = %d, want 0", candidate.Index)
|
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
|
||||||
}
|
}
|
||||||
if candidate.ExtractorKey != extractor.Key() {
|
if candidate.ExtractorKey != extractor.Key() {
|
||||||
t.Fatalf("Candidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
|
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
|
||||||
}
|
}
|
||||||
if candidate.ArtifactType != extractor.ArtifactType() {
|
if candidate.ArtifactType != extractor.ArtifactType() {
|
||||||
t.Fatalf("Candidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
|
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
|
||||||
}
|
}
|
||||||
if candidate.SchemaVersion != extractor.SchemaVersion() {
|
if candidate.SchemaVersion != extractor.SchemaVersion() {
|
||||||
t.Fatalf("Candidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
|
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
|
||||||
}
|
}
|
||||||
if string(candidate.Payload) != `{"value":"example"}` {
|
if string(candidate.Payload) != `{"value":"example"}` {
|
||||||
t.Fatalf("Candidate.Payload = %s, want example payload", candidate.Payload)
|
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,7 +112,7 @@ func (extractor fakeExtractor) Validators() []Validator {
|
|||||||
|
|
||||||
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
|
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
|
||||||
return ExtractionResult{
|
return ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{
|
{
|
||||||
Index: 0,
|
Index: 0,
|
||||||
ExtractorKey: extractor.key,
|
ExtractorKey: extractor.key,
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ func (extractor integrationExtractor) Validators() []contracts.Validator {
|
|||||||
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||||
*extractor.executedKeys = append(*extractor.executedKeys, extractor.key)
|
*extractor.executedKeys = append(*extractor.executedKeys, extractor.key)
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: []artifacts.Candidate{
|
Candidates: []artifacts.ArtifactCandidate{
|
||||||
{Payload: []byte(`{"value":true}`)},
|
{Payload: []byte(`{"value":true}`)},
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
@@ -90,8 +90,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.Candidate, nextIndex *int) ([]artifacts.Candidate, error) {
|
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate, nextIndex *int) ([]artifacts.ArtifactCandidate, error) {
|
||||||
normalized := make([]artifacts.Candidate, 0, len(candidates))
|
normalized := make([]artifacts.ArtifactCandidate, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
candidate.Index = *nextIndex
|
candidate.Index = *nextIndex
|
||||||
*nextIndex = *nextIndex + 1
|
*nextIndex = *nextIndex + 1
|
||||||
@@ -119,7 +119,7 @@ func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.C
|
|||||||
return normalized, nil
|
return normalized, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runValidators(ctx context.Context, extractor contracts.Extractor, doc *source.SourceDocument, candidates []artifacts.Candidate, metadata map[string]any) ([]artifacts.Candidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
|
func runValidators(ctx context.Context, extractor contracts.Extractor, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
|
||||||
eligible := candidates
|
eligible := candidates
|
||||||
var rejected []artifacts.RejectedArtifact
|
var rejected []artifacts.RejectedArtifact
|
||||||
var warnings []contracts.Warning
|
var warnings []contracts.Warning
|
||||||
@@ -149,7 +149,7 @@ func runValidators(ctx context.Context, extractor contracts.Extractor, doc *sour
|
|||||||
decisions[decision.CandidateIndex] = decision
|
decisions[decision.CandidateIndex] = decision
|
||||||
}
|
}
|
||||||
|
|
||||||
nextEligible := make([]artifacts.Candidate, 0, len(eligible))
|
nextEligible := make([]artifacts.ArtifactCandidate, 0, len(eligible))
|
||||||
for _, candidate := range eligible {
|
for _, candidate := range eligible {
|
||||||
decision := decisions[candidate.Index]
|
decision := decisions[candidate.Index]
|
||||||
if decision.Approved {
|
if decision.Approved {
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ func TestRunRejectsNilExtractorFromFactory(t *testing.T) {
|
|||||||
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
||||||
var order []string
|
var order []string
|
||||||
var seenIndices []int
|
var seenIndices []int
|
||||||
recordIndices := func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
recordIndices := func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
seenIndices = append(seenIndices, candidate.Index)
|
seenIndices = append(seenIndices, candidate.Index)
|
||||||
@@ -133,7 +133,7 @@ func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{{Payload: []byte(`{"value":true}`)}}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.ArtifactCandidate{{Payload: []byte(`{"value":true}`)}}},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||||
@@ -150,18 +150,18 @@ func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
|||||||
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
candidate artifacts.Candidate
|
candidate artifacts.ArtifactCandidate
|
||||||
error string
|
error string
|
||||||
}{
|
}{
|
||||||
{name: "extractor key", candidate: artifacts.Candidate{ExtractorKey: "other"}, error: "extractor_key"},
|
{name: "extractor key", candidate: artifacts.ArtifactCandidate{ExtractorKey: "other"}, error: "extractor_key"},
|
||||||
{name: "artifact type", candidate: artifacts.Candidate{ArtifactType: "other"}, error: "artifact_type"},
|
{name: "artifact type", candidate: artifacts.ArtifactCandidate{ArtifactType: "other"}, error: "artifact_type"},
|
||||||
{name: "schema version", candidate: artifacts.Candidate{SchemaVersion: "other"}, error: "schema_version"},
|
{name: "schema version", candidate: artifacts.ArtifactCandidate{SchemaVersion: "other"}, error: "schema_version"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{tt.candidate}},
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.ArtifactCandidate{tt.candidate}},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||||
@@ -194,7 +194,7 @@ func TestRunApprovesCandidatesWithoutValidators(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
||||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||||
}}
|
}}
|
||||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||||
@@ -212,13 +212,13 @@ func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
||||||
var laterSeen int
|
var laterSeen int
|
||||||
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return []contracts.ValidationDecision{
|
return []contracts.ValidationDecision{
|
||||||
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
||||||
validationhelpers.Approved(candidates[1].Index),
|
validationhelpers.Approved(candidates[1].Index),
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
laterSeen = len(candidates)
|
laterSeen = len(candidates)
|
||||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||||
}}
|
}}
|
||||||
@@ -254,7 +254,7 @@ func TestRunSurfacesValidatorNameMismatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
||||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
return nil
|
return nil
|
||||||
}}
|
}}
|
||||||
factory := factoryWithValidator(validator)
|
factory := factoryWithValidator(validator)
|
||||||
@@ -359,7 +359,7 @@ type fakeExtractor struct {
|
|||||||
artifactType string
|
artifactType string
|
||||||
schemaVersion string
|
schemaVersion string
|
||||||
candidateCount int
|
candidateCount int
|
||||||
candidates []artifacts.Candidate
|
candidates []artifacts.ArtifactCandidate
|
||||||
validators []contracts.Validator
|
validators []contracts.Validator
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
@@ -386,9 +386,9 @@ func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.Extrac
|
|||||||
if extractor.order != nil {
|
if extractor.order != nil {
|
||||||
*extractor.order = append(*extractor.order, extractor.key)
|
*extractor.order = append(*extractor.order, extractor.key)
|
||||||
}
|
}
|
||||||
candidates := append([]artifacts.Candidate(nil), extractor.candidates...)
|
candidates := append([]artifacts.ArtifactCandidate(nil), extractor.candidates...)
|
||||||
for len(candidates) < extractor.candidateCount {
|
for len(candidates) < extractor.candidateCount {
|
||||||
candidates = append(candidates, artifacts.Candidate{Payload: []byte(`{"value":true}`)})
|
candidates = append(candidates, artifacts.ArtifactCandidate{Payload: []byte(`{"value":true}`)})
|
||||||
}
|
}
|
||||||
return contracts.ExtractionResult{
|
return contracts.ExtractionResult{
|
||||||
Candidates: candidates,
|
Candidates: candidates,
|
||||||
@@ -399,7 +399,7 @@ func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.Extrac
|
|||||||
type fakeValidator struct {
|
type fakeValidator struct {
|
||||||
name string
|
name string
|
||||||
resultName string
|
resultName string
|
||||||
decisions func([]artifacts.Candidate) []contracts.ValidationDecision
|
decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
@@ -436,7 +436,7 @@ func factoryWithValidator(validator contracts.Validator) fakeFactory {
|
|||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func approveAll(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
|
||||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ func Rejected(candidateIndex int, reasonCode string, message string) contracts.V
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func EnforceDecisionCardinality(candidates []artifacts.Candidate, decisions []contracts.ValidationDecision) error {
|
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error {
|
||||||
if len(candidates) != len(decisions) {
|
if len(candidates) != len(decisions) {
|
||||||
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
|
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func TestRejectedTrimsReasonAndMessage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
|
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
|
||||||
candidates := []artifacts.Candidate{{Index: 4}, {Index: 8}}
|
candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}}
|
||||||
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
|
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
|
||||||
|
|
||||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
||||||
@@ -59,7 +59,7 @@ func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}},
|
[]artifacts.ArtifactCandidate{{Index: 1}},
|
||||||
[]contracts.ValidationDecision{Approved(2)},
|
[]contracts.ValidationDecision{Approved(2)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -68,7 +68,7 @@ func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 2}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -77,7 +77,7 @@ func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 2}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
|
||||||
[]contracts.ValidationDecision{Approved(1)},
|
[]contracts.ValidationDecision{Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,7 +86,7 @@ func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
|
|||||||
|
|
||||||
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
|
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
|
||||||
err := EnforceDecisionCardinality(
|
err := EnforceDecisionCardinality(
|
||||||
[]artifacts.Candidate{{Index: 1}, {Index: 1}},
|
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}},
|
||||||
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
[]contracts.ValidationDecision{Approved(1), Approved(1)},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user